chore: import upstream snapshot with attribution

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/pom.xml

@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ /* ******************************************************************************
+  ~  *
+  ~  *
+  ~  * This program and the accompanying materials are made available under the
+  ~  * terms of the Apache License, Version 2.0 which is available at
+  ~  * https://www.apache.org/licenses/LICENSE-2.0.
+  ~  *
+  ~  *  See the NOTICE file distributed with this work for additional
+  ~  *  information regarding copyright ownership.
+  ~  * Unless required by applicable law or agreed to in writing, software
+  ~  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  ~  * License for the specific language governing permissions and limitations
+  ~  * under the License.
+  ~  *
+  ~  * SPDX-License-Identifier: Apache-2.0
+  ~  ******************************************************************************/
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.eclipse.deeplearning4j</groupId>
+        <artifactId>deeplearning4j-nlp-parent</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>deeplearning4j-nlp</artifactId>
+
+    <properties>
+        <jfasttext.version>0.4</jfasttext.version>
+        <module.name>deeplearning4j.nlp</module.name>
+    </properties>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.moditect</groupId>
+                <artifactId>moditect-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>nd4j-native-api</artifactId>
+            <version>${nd4j.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+            <version>${commons-lang.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>deeplearning4j-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.threadly</groupId>
+            <artifactId>threadly</artifactId>
+            <version>${threadly.version}</version>
+        </dependency>
+
+
+
+
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>${commonslang.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.github.vinhkhuc</groupId>
+            <artifactId>jfasttext</artifactId>
+            <version>${jfasttext.version}</version>
+        </dependency>
+    </dependencies>
+
+
+</project>

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/BagOfWordsVectorizer.java

@@ -0,0 +1,206 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.bagofwords.vectorizer;
+
+import lombok.NonNull;
+import org.apache.commons.io.FileUtils;
+import org.deeplearning4j.models.word2vec.VocabWord;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.deeplearning4j.models.word2vec.wordstore.inmemory.AbstractCache;
+import org.deeplearning4j.text.documentiterator.DocumentIterator;
+import org.deeplearning4j.text.documentiterator.LabelAwareIterator;
+import org.deeplearning4j.text.documentiterator.LabelsSource;
+import org.deeplearning4j.text.documentiterator.interoperability.DocumentIteratorConverter;
+import org.deeplearning4j.text.sentenceiterator.SentenceIterator;
+import org.deeplearning4j.text.sentenceiterator.interoperability.SentenceIteratorConverter;
+import org.deeplearning4j.text.tokenization.tokenizer.Tokenizer;
+import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.util.FeatureUtil;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class BagOfWordsVectorizer extends BaseTextVectorizer {
+
+    protected BagOfWordsVectorizer() {
+
+    }
+
+    /**
+     * Text coming from an input stream considered as one document
+     *
+     * @param is    the input stream to read from
+     * @param label the label to assign
+     * @return a dataset with a applyTransformToDestination of weights(relative to impl; could be word counts or tfidf scores)
+     */
+    @Override
+    public DataSet vectorize(InputStream is, String label) {
+        try {
+            BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
+            String line = "";
+            StringBuilder builder = new StringBuilder();
+            while ((line = reader.readLine()) != null) {
+                builder.append(line);
+            }
+            return vectorize(builder.toString(), label);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public DataSet vectorize(String text, String label) {
+        INDArray input = transform(text);
+        INDArray labelMatrix = FeatureUtil.toOutcomeVector(labelsSource.indexOf(label), labelsSource.size());
+
+        return new DataSet(input, labelMatrix);
+    }
+
+    @Override
+    public INDArray transform(String text) {
+        Tokenizer tokenizer = tokenizerFactory.create(text);
+        List<String> tokens = tokenizer.getTokens();
+        return transform(tokens);
+    }
+
+    @Override
+    public INDArray transform(List<String> tokens) {
+        INDArray input = Nd4j.create(1, vocabCache.numWords());
+        for (String token : tokens) {
+            int idx = vocabCache.indexOf(token);
+            if (vocabCache.indexOf(token) >= 0)
+                input.putScalar(idx, vocabCache.wordFrequency(token));
+        }
+        return input;
+    }
+
+    /**
+     * @param input the text to vectorize
+     * @param label the label of the text
+     * @return {@link DataSet} with a applyTransformToDestination of
+     * weights(relative to impl; could be word counts or tfidf scores)
+     */
+    @Override
+    public DataSet vectorize(File input, String label) {
+        try {
+            String string = FileUtils.readFileToString(input);
+            return vectorize(string, label);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Vectorizes the input source in to a dataset
+     *
+     * @return Adam Gibson
+     */
+    @Override
+    public DataSet vectorize() {
+        throw new UnsupportedOperationException("Can't vectorize empty input");
+    }
+
+    public static class Builder {
+        protected TokenizerFactory tokenizerFactory;
+        protected LabelAwareIterator iterator;
+        protected int minWordFrequency;
+        protected VocabCache<VocabWord> vocabCache;
+        protected LabelsSource labelsSource = new LabelsSource();
+        protected Collection<String> stopWords = new ArrayList<>();
+        protected boolean isParallel = true;
+
+        public Builder() {}
+
+
+        public Builder allowParallelTokenization(boolean reallyAllow) {
+            this.isParallel = reallyAllow;
+            return this;
+        }
+
+        public Builder setTokenizerFactory(@NonNull TokenizerFactory tokenizerFactory) {
+            this.tokenizerFactory = tokenizerFactory;
+            return this;
+        }
+
+        public Builder setIterator(@NonNull LabelAwareIterator iterator) {
+            this.iterator = iterator;
+            return this;
+        }
+
+        public Builder setIterator(@NonNull DocumentIterator iterator) {
+            this.iterator = new DocumentIteratorConverter(iterator, labelsSource);
+            return this;
+        }
+
+        public Builder setIterator(@NonNull SentenceIterator iterator) {
+            this.iterator = new SentenceIteratorConverter(iterator, labelsSource);
+            return this;
+        }
+
+        public Builder setVocab(@NonNull VocabCache<VocabWord> vocab) {
+            this.vocabCache = vocab;
+            return this;
+        }
+
+        public Builder setMinWordFrequency(int minWordFrequency) {
+            this.minWordFrequency = minWordFrequency;
+            return this;
+        }
+
+        public Builder setStopWords(Collection<String> stopWords) {
+            this.stopWords = stopWords;
+            return this;
+        }
+
+        public Builder labelsSource(@NonNull LabelsSource source) {
+            this.labelsSource = source;
+            return this;
+        }
+
+        public BagOfWordsVectorizer build() {
+            BagOfWordsVectorizer vectorizer = new BagOfWordsVectorizer();
+
+            vectorizer.tokenizerFactory = this.tokenizerFactory;
+            vectorizer.iterator = this.iterator;
+            vectorizer.minWordFrequency = this.minWordFrequency;
+            vectorizer.labelsSource = this.labelsSource;
+            vectorizer.stopWords = this.stopWords;
+            vectorizer.isParallel = this.isParallel;
+
+            if (this.vocabCache == null) {
+                this.vocabCache = new AbstractCache.Builder<VocabWord>().build();
+            }
+
+            vectorizer.vocabCache = this.vocabCache;
+
+            return vectorizer;
+        }
+
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/BaseTextVectorizer.java

@@ -0,0 +1,89 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.bagofwords.vectorizer;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.deeplearning4j.models.sequencevectors.iterators.AbstractSequenceIterator;
+import org.deeplearning4j.models.sequencevectors.transformers.impl.SentenceTransformer;
+import org.deeplearning4j.models.word2vec.VocabWord;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.deeplearning4j.models.word2vec.wordstore.VocabConstructor;
+import org.deeplearning4j.models.word2vec.wordstore.inmemory.AbstractCache;
+import org.deeplearning4j.text.documentiterator.LabelAwareIterator;
+import org.deeplearning4j.text.documentiterator.LabelsSource;
+import org.deeplearning4j.text.invertedindex.InvertedIndex;
+import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+public abstract class BaseTextVectorizer implements TextVectorizer {
+    @Setter
+    protected transient TokenizerFactory tokenizerFactory;
+    protected transient LabelAwareIterator iterator;
+    protected int minWordFrequency;
+    @Getter
+    protected VocabCache<VocabWord> vocabCache;
+    protected LabelsSource labelsSource;
+    protected Collection<String> stopWords = new ArrayList<>();
+    @Getter
+    protected transient InvertedIndex<VocabWord> index;
+    @Getter
+    protected boolean isParallel = true;
+
+    public LabelsSource getLabelsSource() {
+        return labelsSource;
+    }
+
+    public void buildVocab() {
+        if (vocabCache == null)
+            vocabCache = new AbstractCache.Builder<VocabWord>().build();
+
+
+        SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(this.iterator)
+                .vocabCache(vocabCache)
+                .tokenizerFactory(tokenizerFactory).build();
+
+        AbstractSequenceIterator<VocabWord> iterator = new AbstractSequenceIterator.Builder<>(transformer).build();
+
+        VocabConstructor<VocabWord> constructor = new VocabConstructor.Builder<VocabWord>()
+                .addSource(iterator, minWordFrequency).setTargetVocabCache(vocabCache).setStopWords(stopWords)
+                .allowParallelTokenization(isParallel).build();
+
+        constructor.buildJointVocabulary(false, true);
+    }
+
+    @Override
+    public void fit() {
+        buildVocab();
+    }
+
+    /**
+     * Returns the number of words encountered so far
+     *
+     * @return the number of words encountered so far
+     */
+    @Override
+    public long numWordsEncountered() {
+        return vocabCache.totalWordOccurrences();
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/DefaultInputStreamCreator.java

@@ -0,0 +1,39 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.bagofwords.vectorizer;
+
+import org.deeplearning4j.models.word2vec.InputStreamCreator;
+import org.deeplearning4j.text.documentiterator.DocumentIterator;
+
+import java.io.InputStream;
+
+public class DefaultInputStreamCreator implements InputStreamCreator {
+    private DocumentIterator iter;
+
+    public DefaultInputStreamCreator(DocumentIterator iter) {
+        this.iter = iter;
+    }
+
+    @Override
+    public InputStream create() {
+        return iter.nextDocument();
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/TextVectorizer.java

@@ -0,0 +1,113 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.bagofwords.vectorizer;
+
+import org.deeplearning4j.core.datasets.vectorizer.Vectorizer;
+import org.deeplearning4j.models.word2vec.VocabWord;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.deeplearning4j.text.invertedindex.InvertedIndex;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.DataSet;
+
+import java.io.File;
+import java.io.InputStream;
+import java.util.List;
+
+public interface TextVectorizer extends Vectorizer {
+
+
+    /**
+     * Sampling for building mini batches
+     * @return the sampling
+     */
+    //double sample();
+
+    /**
+     * For word vectors, this is the batch size for how to partition documents
+     * in to workloads
+     * @return the batchsize for partitioning documents in to workloads
+     */
+    //int batchSize();
+
+    /**
+     * The vocab sorted in descending order
+     * @return the vocab sorted in descending order
+     */
+    VocabCache<VocabWord> getVocabCache();
+
+
+    /**
+     * Text coming from an input stream considered as one document
+     * @param is the input stream to read from
+     * @param label the label to assign
+     * @return a dataset with a applyTransformToDestination of weights(relative to impl; could be word counts or tfidf scores)
+     */
+    DataSet vectorize(InputStream is, String label);
+
+    /**
+     * Vectorizes the passed in text treating it as one document
+     * @param text the text to vectorize
+     * @param label the label of the text
+     * @return a dataset with a transform of weights(relative to impl; could be word counts or tfidf scores)
+     */
+    DataSet vectorize(String text, String label);
+
+    /**
+     * Train the model
+     */
+    void fit();
+
+    /**
+     *
+     * @param input the text to vectorize
+     * @param label the label of the text
+     * @return {@link DataSet} with a applyTransformToDestination of
+     *          weights(relative to impl; could be word counts or tfidf scores)
+     */
+    DataSet vectorize(File input, String label);
+
+
+    /**
+     * Transforms the matrix
+     * @param text text to transform
+     * @return {@link INDArray}
+     */
+    INDArray transform(String text);
+
+    /**
+     * Transforms the matrix
+     * @param tokens
+     * @return
+     */
+    INDArray transform(List<String> tokens);
+
+    /**
+     * Returns the number of words encountered so far
+     * @return the number of words encountered so far
+     */
+    long numWordsEncountered();
+
+    /**
+     * Inverted index
+     * @return the inverted index for this vectorizer
+     */
+    InvertedIndex<VocabWord> getIndex();
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/TfidfVectorizer.java

@@ -0,0 +1,240 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.bagofwords.vectorizer;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.deeplearning4j.models.word2vec.VocabWord;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.deeplearning4j.models.word2vec.wordstore.inmemory.AbstractCache;
+import org.deeplearning4j.text.documentiterator.DocumentIterator;
+import org.deeplearning4j.text.documentiterator.LabelAwareIterator;
+import org.deeplearning4j.text.documentiterator.LabelAwareIteratorWrapper;
+import org.deeplearning4j.text.documentiterator.LabelsSource;
+import org.deeplearning4j.text.documentiterator.interoperability.DocumentIteratorConverter;
+import org.deeplearning4j.text.sentenceiterator.SentenceIterator;
+import org.deeplearning4j.text.sentenceiterator.interoperability.SentenceIteratorConverter;
+import org.deeplearning4j.text.tokenization.tokenizer.Tokenizer;
+import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
+import org.nd4j.common.util.MathUtils;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.util.FeatureUtil;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Slf4j
+public class TfidfVectorizer extends BaseTextVectorizer {
+    /**
+     * Text coming from an input stream considered as one document
+     *
+     * @param is    the input stream to read from
+     * @param label the label to assign
+     * @return a dataset with a applyTransformToDestination of weights(relative to impl; could be word counts or tfidf scores)
+     */
+    @Override
+    public DataSet vectorize(InputStream is, String label) {
+        try {
+            BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
+            String line = "";
+            StringBuilder builder = new StringBuilder();
+            while ((line = reader.readLine()) != null) {
+                builder.append(line);
+            }
+            return vectorize(builder.toString(), label);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Vectorizes the passed in text treating it as one document
+     *
+     * @param text  the text to vectorize
+     * @param label the label of the text
+     * @return a dataset with a transform of weights(relative to impl; could be word counts or tfidf scores)
+     */
+    @Override
+    public DataSet vectorize(String text, String label) {
+        INDArray input = transform(text);
+        INDArray labelMatrix = FeatureUtil.toOutcomeVector(labelsSource.indexOf(label), labelsSource.size());
+
+        return new DataSet(input, labelMatrix);
+    }
+
+    /**
+     * @param input the text to vectorize
+     * @param label the label of the text
+     * @return {@link DataSet} with a applyTransformToDestination of
+     * weights(relative to impl; could be word counts or tfidf scores)
+     */
+    @Override
+    public DataSet vectorize(File input, String label) {
+        try {
+            String string = FileUtils.readFileToString(input);
+            return vectorize(string, label);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Transforms the matrix
+     *
+     * @param text text to transform
+     * @return {@link INDArray}
+     */
+    @Override
+    public INDArray transform(String text) {
+        Tokenizer tokenizer = tokenizerFactory.create(text);
+        List<String> tokens = tokenizer.getTokens();
+
+        // build document words count
+        return transform(tokens);
+    }
+
+
+    @Override
+    public INDArray transform(List<String> tokens) {
+        INDArray ret = Nd4j.create(1, vocabCache.numWords());
+
+        Map<String, AtomicLong> counts = new HashMap<>();
+        for (String token : tokens) {
+            if (!counts.containsKey(token))
+                counts.put(token, new AtomicLong(0));
+
+            counts.get(token).incrementAndGet();
+        }
+
+        for (int i = 0; i < tokens.size(); i++) {
+            int idx = vocabCache.indexOf(tokens.get(i));
+            if (idx >= 0) {
+                double tf_idf = tfidfWord(tokens.get(i), counts.get(tokens.get(i)).longValue(), tokens.size());
+                //log.info("TF-IDF for word: {} -> {} / {} => {}", tokens.get(i), counts.get(tokens.get(i)).longValue(), tokens.size(), tf_idf);
+                ret.putScalar(idx, tf_idf);
+            }
+        }
+        return ret;
+    }
+
+    public double tfidfWord(String word, long wordCount, long documentLength) {
+        //log.info("word: {}; TF: {}; IDF: {}", word, tfForWord(wordCount, documentLength), idfForWord(word));
+        return MathUtils.tfidf(tfForWord(wordCount, documentLength), idfForWord(word));
+    }
+
+    private double tfForWord(long wordCount, long documentLength) {
+        return (double) wordCount / (double) documentLength;
+    }
+
+    private double idfForWord(String word) {
+        return MathUtils.idf(vocabCache.totalNumberOfDocs(), vocabCache.docAppearedIn(word));
+    }
+
+
+    /**
+     * Vectorizes the input source in to a dataset
+     *
+     * @return Adam Gibson
+     */
+    @Override
+    public DataSet vectorize() {
+        return null;
+    }
+
+    public static class Builder {
+        protected TokenizerFactory tokenizerFactory;
+        protected LabelAwareIterator iterator;
+        protected int minWordFrequency;
+        protected VocabCache<VocabWord> vocabCache;
+        protected LabelsSource labelsSource = new LabelsSource();
+        protected Collection<String> stopWords = new ArrayList<>();
+        protected boolean isParallel = true;
+
+        public Builder() {}
+
+        public Builder allowParallelTokenization(boolean reallyAllow) {
+            this.isParallel = reallyAllow;
+            return this;
+        }
+
+        public Builder setTokenizerFactory(@NonNull TokenizerFactory tokenizerFactory) {
+            this.tokenizerFactory = tokenizerFactory;
+            return this;
+        }
+
+        public Builder setIterator(@NonNull LabelAwareIterator iterator) {
+            this.iterator = new LabelAwareIteratorWrapper(iterator, labelsSource);
+            return this;
+        }
+
+        public Builder setIterator(@NonNull DocumentIterator iterator) {
+            this.iterator = new DocumentIteratorConverter(iterator, labelsSource);
+            return this;
+        }
+
+        public Builder setIterator(@NonNull SentenceIterator iterator) {
+            this.iterator = new SentenceIteratorConverter(iterator, labelsSource);
+            return this;
+        }
+
+        public Builder setVocab(@NonNull VocabCache<VocabWord> vocab) {
+            this.vocabCache = vocab;
+            return this;
+        }
+
+        public Builder setMinWordFrequency(int minWordFrequency) {
+            this.minWordFrequency = minWordFrequency;
+            return this;
+        }
+
+        public Builder setStopWords(Collection<String> stopWords) {
+            this.stopWords = stopWords;
+            return this;
+        }
+
+        public TfidfVectorizer build() {
+            TfidfVectorizer vectorizer = new TfidfVectorizer();
+
+            vectorizer.tokenizerFactory = this.tokenizerFactory;
+            vectorizer.iterator = this.iterator;
+            vectorizer.minWordFrequency = this.minWordFrequency;
+            vectorizer.labelsSource = this.labelsSource;
+            vectorizer.isParallel = this.isParallel;
+
+            if (this.vocabCache == null) {
+                this.vocabCache = new AbstractCache.Builder<VocabWord>().build();
+            }
+
+            vectorizer.vocabCache = this.vocabCache;
+            vectorizer.stopWords = this.stopWords;
+
+            return vectorizer;
+        }
+
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/BertIterator.java

@@ -0,0 +1,722 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.iterator;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+import org.deeplearning4j.iterator.bert.BertMaskedLMMasker;
+import org.deeplearning4j.iterator.bert.BertSequenceMasker;
+import org.deeplearning4j.text.tokenization.tokenizer.Tokenizer;
+import org.deeplearning4j.text.tokenization.tokenizerfactory.BertWordPieceTokenizerFactory;
+import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.linalg.api.buffer.DataType;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.indexing.NDArrayIndex;
+import org.nd4j.common.primitives.Pair;
+import org.nd4j.common.primitives.Triple;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+public class BertIterator implements MultiDataSetIterator {
+
+    public enum Task {UNSUPERVISED, SEQ_CLASSIFICATION}
+
+    public enum LengthHandling {FIXED_LENGTH, ANY_LENGTH, CLIP_ONLY}
+
+    public enum FeatureArrays {INDICES_MASK, INDICES_MASK_SEGMENTID}
+
+    public enum UnsupervisedLabelFormat {RANK2_IDX, RANK3_NCL, RANK3_LNC}
+
+    protected Task task;
+    protected TokenizerFactory tokenizerFactory;
+    protected int maxTokens = -1;
+    protected int minibatchSize = 32;
+    protected boolean padMinibatches = false;
+    @Getter
+    @Setter
+    protected MultiDataSetPreProcessor preProcessor;
+    protected LabeledSentenceProvider sentenceProvider = null;
+    protected LabeledPairSentenceProvider sentencePairProvider = null;
+    protected LengthHandling lengthHandling;
+    protected FeatureArrays featureArrays;
+    protected Map<String, Integer> vocabMap;   //TODO maybe use Eclipse ObjectIntHashMap or similar for fewer objects?
+    protected BertSequenceMasker masker = null;
+    protected UnsupervisedLabelFormat unsupervisedLabelFormat = null;
+    protected String maskToken;
+    protected String prependToken;
+    protected String appendToken;
+
+
+    protected List<String> vocabKeysAsList;
+
+    protected BertIterator(Builder b) {
+        this.task = b.task;
+        this.tokenizerFactory = b.tokenizerFactory;
+        this.maxTokens = b.maxTokens;
+        this.minibatchSize = b.minibatchSize;
+        this.padMinibatches = b.padMinibatches;
+        this.preProcessor = b.preProcessor;
+        this.sentenceProvider = b.sentenceProvider;
+        this.sentencePairProvider = b.sentencePairProvider;
+        this.lengthHandling = b.lengthHandling;
+        this.featureArrays = b.featureArrays;
+        this.vocabMap = b.vocabMap;
+        this.masker = b.masker;
+        this.unsupervisedLabelFormat = b.unsupervisedLabelFormat;
+        this.maskToken = b.maskToken;
+        this.prependToken = b.prependToken;
+        this.appendToken = b.appendToken;
+    }
+
+    @Override
+    public boolean hasNext() {
+        if (sentenceProvider != null)
+            return sentenceProvider.hasNext();
+        return sentencePairProvider.hasNext();
+    }
+
+    @Override
+    public MultiDataSet next() {
+        return next(minibatchSize);
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public MultiDataSet next(int num) {
+        Preconditions.checkState(hasNext(), "No next element available");
+        List<Pair<List<String>, String>> tokensAndLabelList;
+        int mbSize = 0;
+        int outLength;
+        long[] segIdOnesFrom = null;
+        if (sentenceProvider != null) {
+            List<Pair<String, String>> list = new ArrayList<>(num);
+            while (sentenceProvider.hasNext() && mbSize++ < num) {
+                list.add(sentenceProvider.nextSentence());
+            }
+            SentenceListProcessed sentenceListProcessed = tokenizeMiniBatch(list);
+            tokensAndLabelList = sentenceListProcessed.getTokensAndLabelList();
+            outLength = sentenceListProcessed.getMaxL();
+        } else if (sentencePairProvider != null) {
+            List<Triple<String, String, String>> listPairs = new ArrayList<>(num);
+            while (sentencePairProvider.hasNext() && mbSize++ < num) {
+                listPairs.add(sentencePairProvider.nextSentencePair());
+            }
+            SentencePairListProcessed sentencePairListProcessed = tokenizePairsMiniBatch(listPairs);
+            tokensAndLabelList = sentencePairListProcessed.getTokensAndLabelList();
+            outLength = sentencePairListProcessed.getMaxL();
+            segIdOnesFrom = sentencePairListProcessed.getSegIdOnesFrom();
+        } else {
+            //TODO - other types of iterators...
+            throw new UnsupportedOperationException("Labelled sentence provider is null and no other iterator types have yet been implemented");
+        }
+
+        Pair<INDArray[], INDArray[]> featuresAndMaskArraysPair = convertMiniBatchFeatures(tokensAndLabelList, outLength, segIdOnesFrom);
+        INDArray[] featureArray = featuresAndMaskArraysPair.getFirst();
+        INDArray[] featureMaskArray = featuresAndMaskArraysPair.getSecond();
+
+
+        Pair<INDArray[], INDArray[]> labelsAndMaskArraysPair = convertMiniBatchLabels(tokensAndLabelList, featureArray, outLength);
+        INDArray[] labelArray = labelsAndMaskArraysPair.getFirst();
+        INDArray[] labelMaskArray = labelsAndMaskArraysPair.getSecond();
+
+        org.nd4j.linalg.dataset.MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(featureArray, labelArray, featureMaskArray, labelMaskArray);
+        if (preProcessor != null)
+            preProcessor.preProcess(mds);
+
+        return mds;
+    }
+
+
+    /**
+     * For use during inference. Will convert a given list of sentences to features and feature masks as appropriate.
+     *
+     * @param listOnlySentences
+     * @return Pair of INDArrays[], first element is feature arrays and the second is the masks array
+     */
+    public Pair<INDArray[], INDArray[]> featurizeSentences(List<String> listOnlySentences) {
+
+        List<Pair<String, String>> sentencesWithNullLabel = addDummyLabel(listOnlySentences);
+        SentenceListProcessed sentenceListProcessed = tokenizeMiniBatch(sentencesWithNullLabel);
+        List<Pair<List<String>, String>> tokensAndLabelList = sentenceListProcessed.getTokensAndLabelList();
+        int outLength = sentenceListProcessed.getMaxL();
+
+        if (preProcessor != null) {
+            Pair<INDArray[], INDArray[]> featureFeatureMasks = convertMiniBatchFeatures(tokensAndLabelList, outLength, null);
+            MultiDataSet dummyMDS = new org.nd4j.linalg.dataset.MultiDataSet(featureFeatureMasks.getFirst(), null, featureFeatureMasks.getSecond(), null);
+            preProcessor.preProcess(dummyMDS);
+            return new Pair<>(dummyMDS.getFeatures(), dummyMDS.getFeaturesMaskArrays());
+        }
+        return convertMiniBatchFeatures(tokensAndLabelList, outLength, null);
+    }
+
+    /**
+     * For use during inference. Will convert a given pair of a list of sentences to features and feature masks as appropriate.
+     *
+     * @param listOnlySentencePairs
+     * @return Pair of INDArrays[], first element is feature arrays and the second is the masks array
+     */
+    public Pair<INDArray[], INDArray[]> featurizeSentencePairs(List<Pair<String, String>> listOnlySentencePairs) {
+        Preconditions.checkState(sentencePairProvider != null, "The featurizeSentencePairs method is meant for inference with sentence pairs. Use only when the sentence pair provider is set (i.e not null).");
+
+        List<Triple<String, String, String>> sentencePairsWithNullLabel = addDummyLabelForPairs(listOnlySentencePairs);
+        SentencePairListProcessed sentencePairListProcessed = tokenizePairsMiniBatch(sentencePairsWithNullLabel);
+        List<Pair<List<String>, String>> tokensAndLabelList = sentencePairListProcessed.getTokensAndLabelList();
+        int outLength = sentencePairListProcessed.getMaxL();
+        long[] segIdOnesFrom = sentencePairListProcessed.getSegIdOnesFrom();
+        if (preProcessor != null) {
+            Pair<INDArray[], INDArray[]> featuresAndMaskArraysPair = convertMiniBatchFeatures(tokensAndLabelList, outLength, segIdOnesFrom);
+            MultiDataSet dummyMDS = new org.nd4j.linalg.dataset.MultiDataSet(featuresAndMaskArraysPair.getFirst(), null, featuresAndMaskArraysPair.getSecond(), null);
+            preProcessor.preProcess(dummyMDS);
+            return new Pair<>(dummyMDS.getFeatures(), dummyMDS.getFeaturesMaskArrays());
+        }
+        return convertMiniBatchFeatures(tokensAndLabelList, outLength, segIdOnesFrom);
+    }
+
+    private Pair<INDArray[], INDArray[]> convertMiniBatchFeatures(List<Pair<List<String>, String>> tokensAndLabelList, int outLength, long[] segIdOnesFrom) {
+        int mbPadded = padMinibatches ? minibatchSize : tokensAndLabelList.size();
+        int[][] outIdxs = new int[mbPadded][outLength];
+        int[][] outMask = new int[mbPadded][outLength];
+        int[][] outSegmentId = null;
+        if (featureArrays == FeatureArrays.INDICES_MASK_SEGMENTID)
+            outSegmentId = new int[mbPadded][outLength];
+        for (int i = 0; i < tokensAndLabelList.size(); i++) {
+            Pair<List<String>, String> p = tokensAndLabelList.get(i);
+            List<String> t = p.getFirst();
+            for (int j = 0; j < outLength && j < t.size(); j++) {
+                Preconditions.checkState(vocabMap.containsKey(t.get(j)), "Unknown token encountered: token \"%s\" is not in vocabulary", t.get(j));
+                int idx = vocabMap.get(t.get(j));
+                outIdxs[i][j] = idx;
+                outMask[i][j] = 1;
+                if (segIdOnesFrom != null && j >= segIdOnesFrom[i])
+                    outSegmentId[i][j] = 1;
+            }
+        }
+
+        //Create actual arrays. Indices, mask, and optional segment ID
+        INDArray outIdxsArr = Nd4j.createFromArray(outIdxs);
+        INDArray outMaskArr = Nd4j.createFromArray(outMask);
+        INDArray outSegmentIdArr;
+        INDArray[] f;
+        INDArray[] fm;
+        if (featureArrays == FeatureArrays.INDICES_MASK_SEGMENTID) {
+            outSegmentIdArr = Nd4j.createFromArray(outSegmentId);
+            f = new INDArray[]{outIdxsArr, outSegmentIdArr};
+            fm = new INDArray[]{outMaskArr, null};
+        } else {
+            f = new INDArray[]{outIdxsArr};
+            fm = new INDArray[]{outMaskArr};
+        }
+        return new Pair<>(f, fm);
+    }
+
+    private SentenceListProcessed tokenizeMiniBatch(List<Pair<String, String>> list) {
+        //Get and tokenize the sentences for this minibatch
+        SentenceListProcessed sentenceListProcessed = new SentenceListProcessed(list.size());
+        int longestSeq = -1;
+        for (Pair<String, String> p : list) {
+            List<String> tokens = tokenizeSentence(p.getFirst());
+            sentenceListProcessed.addProcessedToList(new Pair<>(tokens, p.getSecond()));
+            longestSeq = Math.max(longestSeq, tokens.size());
+        }
+        //Determine output array length...
+        int outLength;
+        switch (lengthHandling) {
+            case FIXED_LENGTH:
+                outLength = maxTokens;
+                break;
+            case ANY_LENGTH:
+                outLength = longestSeq;
+                break;
+            case CLIP_ONLY:
+                outLength = Math.min(maxTokens, longestSeq);
+                break;
+            default:
+                throw new RuntimeException("Not implemented length handling mode: " + lengthHandling);
+        }
+        sentenceListProcessed.setMaxL(outLength);
+        return sentenceListProcessed;
+    }
+
+    private SentencePairListProcessed tokenizePairsMiniBatch(List<Triple<String, String, String>> listPairs) {
+        SentencePairListProcessed sentencePairListProcessed = new SentencePairListProcessed(listPairs.size());
+        for (Triple<String, String, String> t : listPairs) {
+            List<String> tokensL = tokenizeSentence(t.getFirst(), true);
+            List<String> tokensR = tokenizeSentence(t.getSecond(), true);
+            List<String> tokens = new ArrayList<>(maxTokens);
+            int maxLength = maxTokens;
+            if (prependToken != null)
+                maxLength--;
+            if (appendToken != null)
+                maxLength -= 2;
+            if (tokensL.size() + tokensR.size() > maxLength) {
+                boolean shortOnL = tokensL.size() < tokensR.size();
+                int shortSize = Math.min(tokensL.size(), tokensR.size());
+                if (shortSize > maxLength / 2) {
+                    //both lists need to be sliced
+                    tokensL.subList(maxLength / 2, tokensL.size()).clear(); //if maxsize/2 is odd pop extra on L side to match implementation in TF
+                    tokensR.subList(maxLength - maxLength / 2, tokensR.size()).clear();
+                } else {
+                    //slice longer list
+                    if (shortOnL) {
+                        //longer on R - slice R
+                        tokensR.subList(maxLength - tokensL.size(), tokensR.size()).clear();
+                    } else {
+                        //longer on L - slice L
+                        tokensL.subList(maxLength - tokensR.size(), tokensL.size()).clear();
+                    }
+                }
+            }
+            if (prependToken != null)
+                tokens.add(prependToken);
+            tokens.addAll(tokensL);
+            if (appendToken != null)
+                tokens.add(appendToken);
+            int segIdOnesFrom = tokens.size();
+            tokens.addAll(tokensR);
+            if (appendToken != null)
+                tokens.add(appendToken);
+            sentencePairListProcessed.addProcessedToList(segIdOnesFrom, new Pair<>(tokens, t.getThird()));
+        }
+        sentencePairListProcessed.setMaxL(maxTokens);
+        return sentencePairListProcessed;
+    }
+
+    private Pair<INDArray[], INDArray[]> convertMiniBatchLabels(List<Pair<List<String>, String>> tokenizedSentences, INDArray[] featureArray, int outLength) {
+        INDArray[] l = new INDArray[1];
+        INDArray[] lm;
+        int mbSize = tokenizedSentences.size();
+        int mbPadded = padMinibatches ? minibatchSize : tokenizedSentences.size();
+        if (task == Task.SEQ_CLASSIFICATION) {
+            //Sequence classification task: output is 2d, one-hot, shape [minibatch, numClasses]
+            int numClasses;
+            int[] classLabels = new int[mbPadded];
+            if (sentenceProvider != null) {
+                numClasses = sentenceProvider.numLabelClasses();
+                List<String> labels = sentenceProvider.allLabels();
+                for (int i = 0; i < mbSize; i++) {
+                    String lbl = tokenizedSentences.get(i).getRight();
+                    classLabels[i] = labels.indexOf(lbl);
+                    Preconditions.checkState(classLabels[i] >= 0, "Provided label \"%s\" for sentence does not exist in set of classes/categories", lbl);
+                }
+            } else if (sentencePairProvider != null) {
+                numClasses = sentencePairProvider.numLabelClasses();
+                List<String> labels = sentencePairProvider.allLabels();
+                for (int i = 0; i < mbSize; i++) {
+                    String lbl = tokenizedSentences.get(i).getRight();
+                    classLabels[i] = labels.indexOf(lbl);
+                    Preconditions.checkState(classLabels[i] >= 0, "Provided label \"%s\" for sentence does not exist in set of classes/categories", lbl);
+                }
+            } else {
+                throw new RuntimeException();
+            }
+            l[0] = Nd4j.create(DataType.FLOAT, mbPadded, numClasses);
+            for (int i = 0; i < mbSize; i++) {
+                l[0].putScalar(i, classLabels[i], 1.0);
+            }
+            lm = null;
+            if (padMinibatches && mbSize != mbPadded) {
+                INDArray a = Nd4j.zeros(DataType.FLOAT, mbPadded, 1);
+                lm = new INDArray[]{a};
+                a.get(NDArrayIndex.interval(0, mbSize), NDArrayIndex.all()).assign(1);
+            }
+        } else if (task == Task.UNSUPERVISED) {
+            //Unsupervised, masked language model task
+            //Output is either 2d, or 3d depending on settings
+            if (vocabKeysAsList == null) {
+                String[] arr = new String[vocabMap.size()];
+                for (Map.Entry<String, Integer> e : vocabMap.entrySet()) {
+                    arr[e.getValue()] = e.getKey();
+                }
+                vocabKeysAsList = Arrays.asList(arr);
+            }
+
+
+            int vocabSize = vocabMap.size();
+            INDArray labelArr;
+            INDArray lMask = Nd4j.zeros(DataType.INT, mbPadded, outLength);
+            if (unsupervisedLabelFormat == UnsupervisedLabelFormat.RANK2_IDX) {
+                labelArr = Nd4j.create(DataType.INT, mbPadded, outLength);
+            } else if (unsupervisedLabelFormat == UnsupervisedLabelFormat.RANK3_NCL) {
+                labelArr = Nd4j.create(DataType.FLOAT, mbPadded, vocabSize, outLength);
+            } else if (unsupervisedLabelFormat == UnsupervisedLabelFormat.RANK3_LNC) {
+                labelArr = Nd4j.create(DataType.FLOAT, outLength, mbPadded, vocabSize);
+            } else {
+                throw new IllegalStateException("Unknown unsupervised label format: " + unsupervisedLabelFormat);
+            }
+
+            for (int i = 0; i < mbSize; i++) {
+                List<String> tokens = tokenizedSentences.get(i).getFirst();
+                Pair<List<String>, boolean[]> p = masker.maskSequence(tokens, maskToken, vocabKeysAsList);
+                List<String> maskedTokens = p.getFirst();
+                boolean[] predictionTarget = p.getSecond();
+                int seqLen = Math.min(predictionTarget.length, outLength);
+                for (int j = 0; j < seqLen; j++) {
+                    if (predictionTarget[j]) {
+                        String oldToken = tokenizedSentences.get(i).getFirst().get(j);  //This is target
+                        int targetTokenIdx = vocabMap.get(oldToken);
+                        if (unsupervisedLabelFormat == UnsupervisedLabelFormat.RANK2_IDX) {
+                            labelArr.putScalar(i, j, targetTokenIdx);
+                        } else if (unsupervisedLabelFormat == UnsupervisedLabelFormat.RANK3_NCL) {
+                            labelArr.putScalar(i, j, targetTokenIdx, 1.0);
+                        } else if (unsupervisedLabelFormat == UnsupervisedLabelFormat.RANK3_LNC) {
+                            labelArr.putScalar(j, i, targetTokenIdx, 1.0);
+                        }
+
+                        lMask.putScalar(i, j, 1.0);
+
+                        //Also update previously created feature label indexes:
+                        String newToken = maskedTokens.get(j);
+                        int newTokenIdx = vocabMap.get(newToken);
+                        //first element of features is outIdxsArr
+                        featureArray[0].putScalar(i, j, newTokenIdx);
+                    }
+                }
+            }
+            l[0] = labelArr;
+            lm = new INDArray[1];
+            lm[0] = lMask;
+        } else {
+            throw new IllegalStateException("Task not yet implemented: " + task);
+        }
+        return new Pair<>(l, lm);
+    }
+
+    private List<String> tokenizeSentence(String sentence) {
+        return tokenizeSentence(sentence, false);
+    }
+
+    private List<String> tokenizeSentence(String sentence, boolean ignorePrependAppend) {
+        Tokenizer t = tokenizerFactory.create(sentence);
+
+        List<String> tokens = new ArrayList<>();
+        if (prependToken != null && !ignorePrependAppend)
+            tokens.add(prependToken);
+
+        while (t.hasMoreTokens()) {
+            String token = t.nextToken();
+            tokens.add(token);
+        }
+        if (appendToken != null && !ignorePrependAppend)
+            tokens.add(appendToken);
+        return tokens;
+    }
+
+
+    private List<Pair<String, String>> addDummyLabel(List<String> listOnlySentences) {
+        List<Pair<String, String>> list = new ArrayList<>(listOnlySentences.size());
+        for (String s : listOnlySentences) {
+            list.add(new Pair<String, String>(s, null));
+        }
+        return list;
+    }
+
+    private List<Triple<String, String, String>> addDummyLabelForPairs(List<Pair<String, String>> listOnlySentencePairs) {
+        List<Triple<String, String, String>> list = new ArrayList<>(listOnlySentencePairs.size());
+        for (Pair<String, String> p : listOnlySentencePairs) {
+            list.add(new Triple<String, String, String>(p.getFirst(), p.getSecond(), null));
+        }
+        return list;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return true;
+    }
+
+    @Override
+    public void reset() {
+        if (sentenceProvider != null) {
+            sentenceProvider.reset();
+        }
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static class Builder {
+
+        protected Task task;
+        protected TokenizerFactory tokenizerFactory;
+        protected LengthHandling lengthHandling = LengthHandling.FIXED_LENGTH;
+        protected int maxTokens = -1;
+        protected int minibatchSize = 32;
+        protected boolean padMinibatches = false;
+        protected MultiDataSetPreProcessor preProcessor;
+        protected LabeledSentenceProvider sentenceProvider = null;
+        protected LabeledPairSentenceProvider sentencePairProvider = null;
+        protected FeatureArrays featureArrays = FeatureArrays.INDICES_MASK_SEGMENTID;
+        protected Map<String, Integer> vocabMap;   //TODO maybe use Eclipse ObjectIntHashMap for fewer objects?
+        protected BertSequenceMasker masker = new BertMaskedLMMasker();
+        protected UnsupervisedLabelFormat unsupervisedLabelFormat;
+        protected String maskToken;
+        protected String prependToken;
+        protected String appendToken;
+
+        /**
+         * Specify the {@link Task} the iterator should be set up for. See {@link BertIterator} for more details.
+         */
+        public Builder task(Task task) {
+            this.task = task;
+            return this;
+        }
+
+        /**
+         * Specify the TokenizerFactory to use.
+         * For BERT, typically {@link org.deeplearning4j.text.tokenization.tokenizerfactory.BertWordPieceTokenizerFactory}
+         * is used
+         */
+        public Builder tokenizer(TokenizerFactory tokenizerFactory) {
+            this.tokenizerFactory = tokenizerFactory;
+            return this;
+        }
+
+        /**
+         * Specifies how the sequence length of the output data should be handled. See {@link BertIterator} for more details.
+         *
+         * @param lengthHandling Length handling
+         * @param maxLength      Not used if LengthHandling is set to {@link LengthHandling#ANY_LENGTH}
+         * @return
+         */
+        public Builder lengthHandling(@NonNull LengthHandling lengthHandling, int maxLength) {
+            this.lengthHandling = lengthHandling;
+            this.maxTokens = maxLength;
+            return this;
+        }
+
+        /**
+         * Minibatch size to use (number of examples to train on for each iteration)
+         * See also: {@link #padMinibatches}
+         *
+         * @param minibatchSize Minibatch size
+         */
+        public Builder minibatchSize(int minibatchSize) {
+            this.minibatchSize = minibatchSize;
+            return this;
+        }
+
+        /**
+         * Default: false (disabled)<br>
+         * If the dataset is not an exact multiple of the minibatch size, should we pad the smaller final minibatch?<br>
+         * For example, if we have 100 examples total, and 32 minibatch size, the following number of examples will be returned
+         * for subsequent calls of next() in the one epoch:<br>
+         * padMinibatches = false (default): 32, 32, 32, 4.<br>
+         * padMinibatches = true: 32, 32, 32, 32 (note: the last minibatch will have 4 real examples, and 28 masked out padding examples).<br>
+         * Both options should result in exactly the same model. However, some BERT implementations may require exactly an
+         * exact number of examples in all minibatches to function.
+         */
+        public Builder padMinibatches(boolean padMinibatches) {
+            this.padMinibatches = padMinibatches;
+            return this;
+        }
+
+        /**
+         * Set the preprocessor to be used on the MultiDataSets before returning them. Default: none (null)
+         */
+        public Builder preProcessor(MultiDataSetPreProcessor preProcessor) {
+            this.preProcessor = preProcessor;
+            return this;
+        }
+
+        /**
+         * Specify the source of the data for classification.
+         */
+        public Builder sentenceProvider(LabeledSentenceProvider sentenceProvider) {
+            this.sentenceProvider = sentenceProvider;
+            return this;
+        }
+
+        /**
+         * Specify the source of the data for classification on sentence pairs.
+         */
+        public Builder sentencePairProvider(LabeledPairSentenceProvider sentencePairProvider) {
+            this.sentencePairProvider = sentencePairProvider;
+            return this;
+        }
+
+        /**
+         * Specify what arrays should be returned. See {@link BertIterator} for more details.
+         */
+        public Builder featureArrays(FeatureArrays featureArrays) {
+            this.featureArrays = featureArrays;
+            return this;
+        }
+
+        /**
+         * Provide the vocabulary as a map. Keys are the words in the vocabulary, and values are the indices of those
+         * words. For indices, they should be in range 0 to vocabMap.size()-1 inclusive.<br>
+         * If using {@link BertWordPieceTokenizerFactory},
+         * this can be obtained using {@link BertWordPieceTokenizerFactory#getVocab()}
+         */
+        public Builder vocabMap(Map<String, Integer> vocabMap) {
+            this.vocabMap = vocabMap;
+            return this;
+        }
+
+        /**
+         * Used only for unsupervised training (i.e., when task is set to {@link Task#UNSUPERVISED} for learning a
+         * masked language model. This can be used to customize how the masking is performed.<br>
+         * Default: {@link BertMaskedLMMasker}
+         */
+        public Builder masker(BertSequenceMasker masker) {
+            this.masker = masker;
+            return this;
+        }
+
+        /**
+         * Used only for unsupervised training (i.e., when task is set to {@link Task#UNSUPERVISED} for learning a
+         * masked language model. Used to specify the format that the labels should be returned in.
+         * See {@link BertIterator} for more details.
+         */
+        public Builder unsupervisedLabelFormat(UnsupervisedLabelFormat labelFormat) {
+            this.unsupervisedLabelFormat = labelFormat;
+            return this;
+        }
+
+        /**
+         * Used only for unsupervised training (i.e., when task is set to {@link Task#UNSUPERVISED} for learning a
+         * masked language model. This specifies the token (such as "[MASK]") that should be used when a value is masked out.
+         * Note that this is passed to the {@link BertSequenceMasker} defined by {@link #masker(BertSequenceMasker)} hence
+         * the exact behaviour will depend on what masker is used.<br>
+         * Note that this must be in the vocabulary map set in {@link #vocabMap}
+         */
+        public Builder maskToken(String maskToken) {
+            this.maskToken = maskToken;
+            return this;
+        }
+
+        /**
+         * Prepend the specified token to the sequences, when doing supervised training.<br>
+         * i.e., any token sequences will have this added at the start.<br>
+         * Some BERT/Transformer models may need this - for example sequences starting with a "[CLS]" token.<br>
+         * No token is prepended by default.
+         *
+         * @param prependToken The token to start each sequence with (null: no token will be prepended)
+         */
+        public Builder prependToken(String prependToken) {
+            this.prependToken = prependToken;
+            return this;
+        }
+
+        /**
+         * Append the specified token to the sequences, when doing training on sentence pairs.<br>
+         * Generally "[SEP]" is used
+         * No token in appended by default.
+         *
+         * @param appendToken Token at end of each sentence for pairs of sentences (null: no token will be appended)
+         * @return
+         */
+        public Builder appendToken(String appendToken) {
+            this.appendToken = appendToken;
+            return this;
+        }
+
+        public BertIterator build() {
+            Preconditions.checkState(task != null, "No task has been set. Use .task(BertIterator.Task.X) to set the task to be performed");
+            Preconditions.checkState(tokenizerFactory != null, "No tokenizer factory has been set. A tokenizer factory (such as BertWordPieceTokenizerFactory) is required");
+            Preconditions.checkState(vocabMap != null, "Cannot create iterator: No vocabMap has been set. Use Builder.vocabMap(Map<String,Integer>) to set");
+            Preconditions.checkState(task != Task.UNSUPERVISED || masker != null, "If task is UNSUPERVISED training, a masker must be set via masker(BertSequenceMasker) method");
+            Preconditions.checkState(task != Task.UNSUPERVISED || unsupervisedLabelFormat != null, "If task is UNSUPERVISED training, a label format must be set via masker(BertSequenceMasker) method");
+            Preconditions.checkState(task != Task.UNSUPERVISED || maskToken != null, "If task is UNSUPERVISED training, the mask token in the vocab (such as \"[MASK]\" must be specified");
+            if (sentencePairProvider != null) {
+                Preconditions.checkState(task == Task.SEQ_CLASSIFICATION, "Currently only supervised sequence classification is set up with sentence pairs. \".task(BertIterator.Task.SEQ_CLASSIFICATION)\" is required with a sentence pair provider");
+                Preconditions.checkState(featureArrays == FeatureArrays.INDICES_MASK_SEGMENTID, "Currently only supervised sequence classification is set up with sentence pairs. \".featureArrays(FeatureArrays.INDICES_MASK_SEGMENTID)\" is required with a sentence pair provider");
+                Preconditions.checkState(lengthHandling == LengthHandling.FIXED_LENGTH, "Currently only fixed length is supported for sentence pairs. \".lengthHandling(BertIterator.LengthHandling.FIXED_LENGTH, maxLength)\" is required with a sentence pair provider");
+                Preconditions.checkState(sentencePairProvider != null, "Provide either a sentence provider or a sentence pair provider. Both cannot be non null");
+            }
+            if (appendToken != null) {
+                Preconditions.checkState(sentencePairProvider != null, "Tokens are only appended with sentence pairs. Sentence pair provider is not set. Set sentence pair provider.");
+            }
+            return new BertIterator(this);
+        }
+    }
+
+    private static class SentencePairListProcessed {
+        private int listLength = 0;
+
+        @Getter
+        private long[] segIdOnesFrom;
+        private int cursor = 0;
+        private SentenceListProcessed sentenceListProcessed;
+
+        private SentencePairListProcessed(int listLength) {
+            this.listLength = listLength;
+            segIdOnesFrom = new long[listLength];
+            sentenceListProcessed = new SentenceListProcessed(listLength);
+        }
+
+        private void addProcessedToList(long segIdIdx, Pair<List<String>, String> tokenizedSentencePairAndLabel) {
+            segIdOnesFrom[cursor] = segIdIdx;
+            sentenceListProcessed.addProcessedToList(tokenizedSentencePairAndLabel);
+            cursor++;
+        }
+
+        private void setMaxL(int maxL) {
+            sentenceListProcessed.setMaxL(maxL);
+        }
+
+        private int getMaxL() {
+            return sentenceListProcessed.getMaxL();
+        }
+
+        private List<Pair<List<String>, String>> getTokensAndLabelList() {
+            return sentenceListProcessed.getTokensAndLabelList();
+        }
+    }
+
+    private static class SentenceListProcessed {
+        private int listLength;
+
+        @Getter
+        @Setter
+        private int maxL;
+
+        @Getter
+        private List<Pair<List<String>, String>> tokensAndLabelList;
+
+        private SentenceListProcessed(int listLength) {
+            this.listLength = listLength;
+            tokensAndLabelList = new ArrayList<>(listLength);
+        }
+
+        private void addProcessedToList(Pair<List<String>, String> tokenizedSentenceAndLabel) {
+            tokensAndLabelList.add(tokenizedSentenceAndLabel);
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/CnnSentenceDataSetIterator.java

@@ -0,0 +1,591 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.iterator;
+
+import lombok.AllArgsConstructor;
+import lombok.NonNull;
+import org.deeplearning4j.iterator.provider.LabelAwareConverter;
+import org.deeplearning4j.models.embeddings.wordvectors.WordVectors;
+import org.deeplearning4j.text.documentiterator.LabelAwareDocumentIterator;
+import org.deeplearning4j.text.documentiterator.LabelAwareIterator;
+import org.deeplearning4j.text.documentiterator.interoperability.DocumentIteratorConverter;
+import org.deeplearning4j.text.sentenceiterator.interoperability.SentenceIteratorConverter;
+import org.deeplearning4j.text.sentenceiterator.labelaware.LabelAwareSentenceIterator;
+import org.deeplearning4j.text.tokenization.tokenizer.Tokenizer;
+import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory;
+import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.indexing.INDArrayIndex;
+import org.nd4j.linalg.indexing.NDArrayIndex;
+import org.nd4j.common.primitives.Pair;
+
+import java.util.*;
+
+@AllArgsConstructor
+public class CnnSentenceDataSetIterator implements DataSetIterator {
+    public enum UnknownWordHandling {
+        RemoveWord, UseUnknownVector
+    }
+
+    /**
+     * Format of features:<br>
+     * CNN1D: For use with 1d convolution layers: Shape [minibatch, vectorSize, sentenceLength]<br>
+     * CNN2D: For use with 2d convolution layers: Shape [minibatch, 1, vectorSize, sentenceLength] or [minibatch, 1, sentenceLength, vectorSize],
+     * depending on the setting for 'sentencesAlongHeight' configuration.
+     */
+    public enum Format {
+        RNN, CNN1D, CNN2D
+    }
+
+    private static final String UNKNOWN_WORD_SENTINEL = "UNKNOWN_WORD_SENTINEL";
+
+    private Format format;
+    private LabeledSentenceProvider sentenceProvider;
+    private WordVectors wordVectors;
+    private TokenizerFactory tokenizerFactory;
+    private UnknownWordHandling unknownWordHandling;
+    private boolean useNormalizedWordVectors;
+    private int minibatchSize;
+    private int maxSentenceLength;
+    private boolean sentencesAlongHeight;
+    private DataSetPreProcessor dataSetPreProcessor;
+
+    private int wordVectorSize;
+    private int numClasses;
+    private Map<String, Integer> labelClassMap;
+    private INDArray unknown;
+
+    private int cursor = 0;
+
+    private Pair<List<String>, String> preLoadedTokens;
+
+    protected CnnSentenceDataSetIterator(Builder builder) {
+        this.format = builder.format;
+        this.sentenceProvider = builder.sentenceProvider;
+        this.wordVectors = builder.wordVectors;
+        this.tokenizerFactory = builder.tokenizerFactory;
+        this.unknownWordHandling = builder.unknownWordHandling;
+        this.useNormalizedWordVectors = builder.useNormalizedWordVectors;
+        this.minibatchSize = builder.minibatchSize;
+        this.maxSentenceLength = builder.maxSentenceLength;
+        this.sentencesAlongHeight = builder.sentencesAlongHeight;
+        this.dataSetPreProcessor = builder.dataSetPreProcessor;
+
+
+        this.numClasses = this.sentenceProvider.numLabelClasses();
+        this.labelClassMap = new HashMap<>();
+        int count = 0;
+        //First: sort the labels to ensure the same label assignment order (say train vs. test)
+        List<String> sortedLabels = new ArrayList<>(this.sentenceProvider.allLabels());
+        Collections.sort(sortedLabels);
+
+        this.wordVectorSize = wordVectors.getWordVector(wordVectors.vocab().wordAtIndex(0)).length;
+
+        for (String s : sortedLabels) {
+            this.labelClassMap.put(s, count++);
+        }
+        if (unknownWordHandling == UnknownWordHandling.UseUnknownVector) {
+            if (useNormalizedWordVectors) {
+                unknown = wordVectors.getWordVectorMatrixNormalized(wordVectors.getUNK());
+            } else {
+                unknown = wordVectors.getWordVectorMatrix(wordVectors.getUNK());
+            }
+
+            if(unknown == null){
+                unknown = wordVectors.getWordVectorMatrix(wordVectors.vocab().wordAtIndex(0)).like();
+            }
+        }
+    }
+
+    /**
+     * Generally used post training time to load a single sentence for predictions
+     */
+    public INDArray loadSingleSentence(String sentence) {
+        List<String> tokens = tokenizeSentence(sentence);
+        if(tokens.isEmpty())
+            throw new IllegalStateException("No tokens available for input sentence - empty string or no words in vocabulary with RemoveWord unknown handling? Sentence = \"" +
+                    sentence + "\"");
+        if(format == Format.CNN1D || format == Format.RNN) {
+            int[] featuresShape = new int[] {1, wordVectorSize, Math.min(maxSentenceLength, tokens.size())};
+            INDArray features = Nd4j.create(featuresShape, (format == Format.CNN1D ? 'c' : 'f'));
+            INDArrayIndex[] indices = new INDArrayIndex[3];
+            indices[0] = NDArrayIndex.point(0);
+            for (int i = 0; i < featuresShape[2]; i++) {
+                INDArray vector = getVector(tokens.get(i));
+                indices[1] = NDArrayIndex.all();
+                indices[2] = NDArrayIndex.point(i);
+                features.put(indices, vector);
+            }
+            return features;
+        } else {
+            int[] featuresShape = new int[] {1, 1, 0, 0};
+            if (sentencesAlongHeight) {
+                featuresShape[2] = Math.min(maxSentenceLength, tokens.size());
+                featuresShape[3] = wordVectorSize;
+            } else {
+                featuresShape[2] = wordVectorSize;
+                featuresShape[3] = Math.min(maxSentenceLength, tokens.size());
+            }
+
+            INDArray features = Nd4j.create(featuresShape);
+            int length = (sentencesAlongHeight ? featuresShape[2] : featuresShape[3]);
+            INDArrayIndex[] indices = new INDArrayIndex[4];
+            indices[0] = NDArrayIndex.point(0);
+            indices[1] = NDArrayIndex.point(0);
+            for (int i = 0; i < length; i++) {
+                INDArray vector = getVector(tokens.get(i));
+
+                if (sentencesAlongHeight) {
+                    indices[2] = NDArrayIndex.point(i);
+                    indices[3] = NDArrayIndex.all();
+                } else {
+                    indices[2] = NDArrayIndex.all();
+                    indices[3] = NDArrayIndex.point(i);
+                }
+
+                features.put(indices, vector);
+            }
+
+            return features;
+        }
+    }
+
+    private INDArray getVector(String word) {
+        INDArray vector;
+        if (unknownWordHandling == UnknownWordHandling.UseUnknownVector && word == UNKNOWN_WORD_SENTINEL) { //Yes, this *should* be using == for the sentinel String here
+            vector = unknown;
+        } else {
+            if (useNormalizedWordVectors) {
+                vector = wordVectors.getWordVectorMatrixNormalized(word);
+            } else {
+                vector = wordVectors.getWordVectorMatrix(word);
+            }
+        }
+        return vector;
+    }
+
+    private List<String> tokenizeSentence(String sentence) {
+        Tokenizer t = tokenizerFactory.create(sentence);
+
+        List<String> tokens = new ArrayList<>();
+        while (t.hasMoreTokens()) {
+            String token = t.nextToken();
+            if (!wordVectors.outOfVocabularySupported() && !wordVectors.hasWord(token)) {
+                switch (unknownWordHandling) {
+                    case RemoveWord:
+                        continue;
+                    case UseUnknownVector:
+                        token = UNKNOWN_WORD_SENTINEL;
+                }
+            }
+            tokens.add(token);
+        }
+        return tokens;
+    }
+
+    public Map<String, Integer> getLabelClassMap() {
+        return new HashMap<>(labelClassMap);
+    }
+
+    @Override
+    public List<String> getLabels() {
+        //We don't want to just return the list from the LabelledSentenceProvider, as we sorted them earlier to do the
+        // String -> Integer mapping
+        String[] str = new String[labelClassMap.size()];
+        for (Map.Entry<String, Integer> e : labelClassMap.entrySet()) {
+            str[e.getValue()] = e.getKey();
+        }
+        return Arrays.asList(str);
+    }
+
+    @Override
+    public boolean hasNext() {
+        if (sentenceProvider == null) {
+            throw new UnsupportedOperationException("Cannot do next/hasNext without a sentence provider");
+        }
+
+        while (preLoadedTokens == null && sentenceProvider.hasNext()) {
+            //Pre-load tokens. Because we filter out empty strings, or sentences with no valid words
+            //we need to pre-load some tokens. Otherwise, sentenceProvider could have 1 (invalid) sentence
+            //next, hasNext() would return true, but next(int) wouldn't be able to return anything
+            preLoadTokens();
+        }
+
+        return preLoadedTokens != null;
+    }
+
+    private void preLoadTokens() {
+        if (preLoadedTokens != null) {
+            return;
+        }
+        Pair<String, String> p = sentenceProvider.nextSentence();
+        List<String> tokens = tokenizeSentence(p.getFirst());
+        if (!tokens.isEmpty()) {
+            preLoadedTokens = new Pair<>(tokens, p.getSecond());
+        }
+    }
+
+    @Override
+    public DataSet next() {
+        return next(minibatchSize);
+    }
+
+    @Override
+    public DataSet next(int num) {
+        if (sentenceProvider == null) {
+            throw new UnsupportedOperationException("Cannot do next/hasNext without a sentence provider");
+        }
+        if (!hasNext()) {
+            throw new NoSuchElementException("No next element");
+        }
+
+
+        List<Pair<List<String>, String>> tokenizedSentences = new ArrayList<>(num);
+        int maxLength = -1;
+        int minLength = Integer.MAX_VALUE; //Track to we know if we can skip mask creation for "all same length" case
+        if (preLoadedTokens != null) {
+            tokenizedSentences.add(preLoadedTokens);
+            maxLength = Math.max(maxLength, preLoadedTokens.getFirst().size());
+            minLength = Math.min(minLength, preLoadedTokens.getFirst().size());
+            preLoadedTokens = null;
+        }
+        for (int i = tokenizedSentences.size(); i < num && sentenceProvider.hasNext(); i++) {
+            Pair<String, String> p = sentenceProvider.nextSentence();
+            List<String> tokens = tokenizeSentence(p.getFirst());
+
+            if (!tokens.isEmpty()) {
+                //Handle edge case: no tokens from sentence
+                maxLength = Math.max(maxLength, tokens.size());
+                minLength = Math.min(minLength, tokens.size());
+                tokenizedSentences.add(new Pair<>(tokens, p.getSecond()));
+            } else {
+                //Skip the current iterator
+                i--;
+            }
+        }
+
+        if (maxSentenceLength > 0 && maxLength > maxSentenceLength) {
+            maxLength = maxSentenceLength;
+        }
+
+        int currMinibatchSize = tokenizedSentences.size();
+        INDArray labels = Nd4j.create(currMinibatchSize, numClasses);
+        for (int i = 0; i < tokenizedSentences.size(); i++) {
+            String labelStr = tokenizedSentences.get(i).getSecond();
+            if (!labelClassMap.containsKey(labelStr)) {
+                throw new IllegalStateException("Got label \"" + labelStr
+                                + "\" that is not present in list of LabeledSentenceProvider labels");
+            }
+
+            int labelIdx = labelClassMap.get(labelStr);
+
+            labels.putScalar(i, labelIdx, 1.0);
+        }
+
+        INDArray features;
+        INDArray featuresMask = null;
+        if(format == Format.CNN1D || format == Format.RNN){
+            int[] featuresShape = new int[]{currMinibatchSize, wordVectorSize, maxLength};
+            features = Nd4j.create(featuresShape, (format == Format.CNN1D ? 'c' : 'f'));
+
+            INDArrayIndex[] idxs = new INDArrayIndex[3];
+            idxs[1] = NDArrayIndex.all();
+            for (int i = 0; i < currMinibatchSize; i++) {
+                idxs[0] = NDArrayIndex.point(i);
+                List<String> currSentence = tokenizedSentences.get(i).getFirst();
+                for (int j = 0; j < currSentence.size() && j < maxSentenceLength; j++) {
+                    idxs[2] = NDArrayIndex.point(j);
+                    INDArray vector = getVector(currSentence.get(j));
+                    features.put(idxs, vector);
+                }
+            }
+
+            if (minLength != maxLength) {
+                featuresMask = Nd4j.create(currMinibatchSize, maxLength);
+                for (int i = 0; i < currMinibatchSize; i++) {
+                    int sentenceLength = tokenizedSentences.get(i).getFirst().size();
+                    if (sentenceLength >= maxLength) {
+                        featuresMask.getRow(i).assign(1.0);
+                    } else {
+                        featuresMask.get(NDArrayIndex.point(i), NDArrayIndex.interval(0, sentenceLength)).assign(1.0);
+                    }
+                }
+            }
+
+        } else {
+            int[] featuresShape = new int[4];
+            featuresShape[0] = currMinibatchSize;
+            featuresShape[1] = 1;
+            if (sentencesAlongHeight) {
+                featuresShape[2] = maxLength;
+                featuresShape[3] = wordVectorSize;
+            } else {
+                featuresShape[2] = wordVectorSize;
+                featuresShape[3] = maxLength;
+            }
+
+            features = Nd4j.create(featuresShape);
+            INDArrayIndex[] indices = new INDArrayIndex[4];
+            indices[1] = NDArrayIndex.point(0);
+            for (int i = 0; i < currMinibatchSize; i++) {
+                indices[0] = NDArrayIndex.point(i);
+                List<String> currSentence = tokenizedSentences.get(i).getFirst();
+                for (int j = 0; j < currSentence.size() && j < maxSentenceLength; j++) {
+                    INDArray vector = getVector(currSentence.get(j));
+
+                    if (sentencesAlongHeight) {
+                        indices[2] = NDArrayIndex.point(j);
+                        indices[3] = NDArrayIndex.all();
+                    } else {
+                        indices[2] = NDArrayIndex.all();
+                        indices[3] = NDArrayIndex.point(j);
+                    }
+
+                    features.put(indices, vector);
+                }
+            }
+
+            if (minLength != maxLength) {
+                if(sentencesAlongHeight){
+                    featuresMask = Nd4j.create(currMinibatchSize, 1, maxLength, 1);
+                    for (int i = 0; i < currMinibatchSize; i++) {
+                        int sentenceLength = tokenizedSentences.get(i).getFirst().size();
+                        if (sentenceLength >= maxLength) {
+                            featuresMask.slice(i).assign(1.0);
+                        } else {
+                            featuresMask.get(NDArrayIndex.point(i), NDArrayIndex.point(0), NDArrayIndex.interval(0, sentenceLength), NDArrayIndex.point(0)).assign(1.0);
+                        }
+                    }
+                } else {
+                    featuresMask = Nd4j.create(currMinibatchSize, 1, 1, maxLength);
+                    for (int i = 0; i < currMinibatchSize; i++) {
+                        int sentenceLength = tokenizedSentences.get(i).getFirst().size();
+                        if (sentenceLength >= maxLength) {
+                            featuresMask.slice(i).assign(1.0);
+                        } else {
+                            featuresMask.get(NDArrayIndex.point(i), NDArrayIndex.point(0), NDArrayIndex.point(0), NDArrayIndex.interval(0, sentenceLength)).assign(1.0);
+                        }
+                    }
+                }
+            }
+        }
+
+        DataSet ds = new DataSet(features, labels, featuresMask, null);
+
+        if (dataSetPreProcessor != null) {
+            dataSetPreProcessor.preProcess(ds);
+        }
+
+        cursor += ds.numExamples();
+        return ds;
+    }
+
+    @Override
+    public int inputColumns() {
+        return wordVectorSize;
+    }
+
+    @Override
+    public int totalOutcomes() {
+        return numClasses;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return true;
+    }
+
+    @Override
+    public void reset() {
+        cursor = 0;
+        sentenceProvider.reset();
+    }
+
+    @Override
+    public int batch() {
+        return minibatchSize;
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        this.dataSetPreProcessor = preProcessor;
+    }
+
+    @Override
+    public DataSetPreProcessor getPreProcessor() {
+        return dataSetPreProcessor;
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    public static class Builder {
+
+        private Format format;
+        private LabeledSentenceProvider sentenceProvider = null;
+        private WordVectors wordVectors;
+        private TokenizerFactory tokenizerFactory = new DefaultTokenizerFactory();
+        private UnknownWordHandling unknownWordHandling = UnknownWordHandling.RemoveWord;
+        private boolean useNormalizedWordVectors = true;
+        private int maxSentenceLength = -1;
+        private int minibatchSize = 32;
+        private boolean sentencesAlongHeight = true;
+        private DataSetPreProcessor dataSetPreProcessor;
+
+        /**
+         * @deprecated Due to old default, that will be changed in the future. Use {@link #Builder(Format)} to specify
+         * the {@link Format} of the activations
+         */
+        @Deprecated
+        public Builder(){
+            //Default for backward compatibility
+            this(Format.CNN2D);
+        }
+
+        /**
+         * @param format The format to use for the features - i.e., for 1D or 2D CNNs
+         */
+        public Builder(@NonNull Format format){
+            this.format = format;
+        }
+
+        /**
+         * Specify how the (labelled) sentences / documents should be provided
+         */
+        public Builder sentenceProvider(LabeledSentenceProvider labeledSentenceProvider) {
+            this.sentenceProvider = labeledSentenceProvider;
+            return this;
+        }
+
+        /**
+         * Specify how the (labelled) sentences / documents should be provided
+         */
+        public Builder sentenceProvider(LabelAwareIterator iterator, @NonNull List<String> labels) {
+            LabelAwareConverter converter = new LabelAwareConverter(iterator, labels);
+            return sentenceProvider(converter);
+        }
+
+        /**
+         * Specify how the (labelled) sentences / documents should be provided
+         */
+        public Builder sentenceProvider(LabelAwareDocumentIterator iterator, @NonNull List<String> labels) {
+            DocumentIteratorConverter converter = new DocumentIteratorConverter(iterator);
+            return sentenceProvider(converter, labels);
+        }
+
+        /**
+         * Specify how the (labelled) sentences / documents should be provided
+         */
+        public Builder sentenceProvider(LabelAwareSentenceIterator iterator, @NonNull List<String> labels) {
+            SentenceIteratorConverter converter = new SentenceIteratorConverter(iterator);
+            return sentenceProvider(converter, labels);
+        }
+
+
+        /**
+         * Provide the WordVectors instance that should be used for training
+         */
+        public Builder wordVectors(WordVectors wordVectors) {
+            this.wordVectors = wordVectors;
+            return this;
+        }
+
+        /**
+         * The {@link TokenizerFactory} that should be used. Defaults to {@link DefaultTokenizerFactory}
+         */
+        public Builder tokenizerFactory(TokenizerFactory tokenizerFactory) {
+            this.tokenizerFactory = tokenizerFactory;
+            return this;
+        }
+
+        /**
+         * Specify how unknown words (those that don't have a word vector in the provided WordVectors instance) should be
+         * handled. Default: remove/ignore unknown words.
+         */
+        public Builder unknownWordHandling(UnknownWordHandling unknownWordHandling) {
+            this.unknownWordHandling = unknownWordHandling;
+            return this;
+        }
+
+        /**
+         * Minibatch size to use for the DataSetIterator
+         */
+        public Builder minibatchSize(int minibatchSize) {
+            this.minibatchSize = minibatchSize;
+            return this;
+        }
+
+        /**
+         * Whether normalized word vectors should be used. Default: true
+         */
+        public Builder useNormalizedWordVectors(boolean useNormalizedWordVectors) {
+            this.useNormalizedWordVectors = useNormalizedWordVectors;
+            return this;
+        }
+
+        /**
+         * Maximum sentence/document length. If sentences exceed this, they will be truncated to this length by
+         * taking the first 'maxSentenceLength' known words.
+         */
+        public Builder maxSentenceLength(int maxSentenceLength) {
+            this.maxSentenceLength = maxSentenceLength;
+            return this;
+        }
+
+        /**
+         * If true (default): output features data with shape [minibatchSize, 1, maxSentenceLength, wordVectorSize]<br>
+         * If false: output features with shape [minibatchSize, 1, wordVectorSize, maxSentenceLength]
+         */
+        public Builder sentencesAlongHeight(boolean sentencesAlongHeight) {
+            this.sentencesAlongHeight = sentencesAlongHeight;
+            return this;
+        }
+
+        /**
+         * Optional DataSetPreProcessor
+         */
+        public Builder dataSetPreProcessor(DataSetPreProcessor dataSetPreProcessor) {
+            this.dataSetPreProcessor = dataSetPreProcessor;
+            return this;
+        }
+
+        public CnnSentenceDataSetIterator build() {
+            if (wordVectors == null) {
+                throw new IllegalStateException(
+                                "Cannot build CnnSentenceDataSetIterator without a WordVectors instance");
+            }
+
+            return new CnnSentenceDataSetIterator(this);
+        }
+
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/LabeledPairSentenceProvider.java

@@ -0,0 +1,61 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.iterator;
+
+import org.nd4j.common.primitives.Triple;
+
+import java.util.List;
+
+public interface LabeledPairSentenceProvider {
+
+    /**
+     * Are there more sentences/documents available?
+     */
+    boolean hasNext();
+
+    /**
+     * @return Triple: two sentence/document texts and label
+     */
+    Triple<String, String, String> nextSentencePair();
+
+    /**
+     * Reset the iterator - including shuffling the order, if necessary/appropriate
+     */
+    void reset();
+
+    /**
+     * Return the total number of sentences, or -1 if not available
+     */
+    int totalNumSentences();
+
+    /**
+     * Return the list of labels - this also defines the class/integer label assignment order
+     */
+    List<String> allLabels();
+
+    /**
+     * Equivalent to allLabels().size()
+     */
+    int numLabelClasses();
+
+}
+
+

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/LabeledSentenceProvider.java

@@ -0,0 +1,60 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.iterator;
+
+import org.nd4j.common.primitives.Pair;
+
+import java.util.List;
+
+public interface LabeledSentenceProvider {
+
+    /**
+     * Are there more sentences/documents available?
+     */
+    boolean hasNext();
+
+    /**
+     *
+     * @return Pair: sentence/document text and label
+     */
+    Pair<String, String> nextSentence();
+
+    /**
+     * Reset the iterator - including shuffling the order, if necessary/appropriate
+     */
+    void reset();
+
+    /**
+     * Return the total number of sentences, or -1 if not available
+     */
+    int totalNumSentences();
+
+    /**
+     * Return the list of labels - this also defines the class/integer label assignment order
+     */
+    List<String> allLabels();
+
+    /**
+     * Equivalent to allLabels().size()
+     */
+    int numLabelClasses();
+
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/bert/BertMaskedLMMasker.java

@@ -0,0 +1,92 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.iterator.bert;
+
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.common.primitives.Pair;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+public class BertMaskedLMMasker implements BertSequenceMasker {
+    public static final double DEFAULT_MASK_PROB = 0.15;
+    public static final double DEFAULT_MASK_TOKEN_PROB = 0.8;
+    public static final double DEFAULT_RANDOM_WORD_PROB = 0.1;
+
+    protected final Random r;
+    protected final double maskProb;
+    protected final double maskTokenProb;
+    protected final double randomTokenProb;
+
+    /**
+     * Create a BertMaskedLMMasker with all default probabilities
+     */
+    public BertMaskedLMMasker(){
+        this(new Random(), DEFAULT_MASK_PROB, DEFAULT_MASK_TOKEN_PROB, DEFAULT_RANDOM_WORD_PROB);
+    }
+
+    /**
+     * See: {@link BertMaskedLMMasker} for details.
+     * @param r                 Random number generator
+     * @param maskProb          Probability of masking each token
+     * @param maskTokenProb     Probability of replacing a selected token with the mask token
+     * @param randomTokenProb    Probability of replacing a selected token with a random token
+     */
+    public BertMaskedLMMasker(Random r, double maskProb, double maskTokenProb, double randomTokenProb){
+        Preconditions.checkArgument(maskProb > 0 && maskProb < 1, "Probability must be beteen 0 and 1, got %s", maskProb);
+        Preconditions.checkState(maskTokenProb >=0 && maskTokenProb <= 1.0, "Mask token probability must be between 0 and 1, got %s", maskTokenProb);
+        Preconditions.checkState(randomTokenProb >=0 && randomTokenProb <= 1.0, "Random token probability must be between 0 and 1, got %s", randomTokenProb);
+        Preconditions.checkState(maskTokenProb + randomTokenProb <= 1.0, "Sum of maskTokenProb (%s) and randomTokenProb (%s) must be <= 1.0, got sum is %s",
+                maskTokenProb, randomTokenProb, (maskTokenProb + randomTokenProb));
+        this.r = r;
+        this.maskProb = maskProb;
+        this.maskTokenProb = maskTokenProb;
+        this.randomTokenProb = randomTokenProb;
+    }
+
+    @Override
+    public Pair<List<String>,boolean[]> maskSequence(List<String> input, String maskToken, List<String> vocabWords) {
+        List<String> out = new ArrayList<>(input.size());
+        boolean[] masked = new boolean[input.size()];
+        for(int i=0; i<input.size(); i++ ){
+            if(r.nextDouble() < maskProb){
+                //Mask
+                double d = r.nextDouble();
+                if(d < maskTokenProb){
+                    out.add(maskToken);
+                } else if(d < maskTokenProb + randomTokenProb){
+                    //Randomly select a token...
+                    String random = vocabWords.get(r.nextInt(vocabWords.size()));
+                    out.add(random);
+                } else {
+                    //Keep existing token
+                    out.add(input.get(i));
+                }
+                masked[i] = true;
+            } else {
+                //No change, keep existing
+                out.add(input.get(i));
+            }
+        }
+        return new Pair<>(out, masked);
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/bert/BertSequenceMasker.java

@@ -0,0 +1,39 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.iterator.bert;
+
+import org.nd4j.common.primitives.Pair;
+
+import java.util.List;
+
+public interface BertSequenceMasker {
+
+    /**
+     *
+     * @param input         Input sequence of tokens
+     * @param maskToken     Token to use for masking - usually something like "[MASK]"
+     * @param vocabWords    Vocabulary, as a list
+     * @return Pair: The new input tokens (after masking out), along with a boolean[] for whether the token is
+     * masked or not (same length as number of tokens). boolean[i] is true if token i was masked.
+     */
+    Pair<List<String>,boolean[]> maskSequence(List<String> input, String maskToken, List<String> vocabWords);
+
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/provider/CollectionLabeledPairSentenceProvider.java

@@ -0,0 +1,134 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.iterator.provider;
+
+import lombok.NonNull;
+import org.deeplearning4j.iterator.LabeledPairSentenceProvider;
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.common.primitives.Triple;
+import org.nd4j.common.util.MathUtils;
+
+import java.util.*;
+
+public class CollectionLabeledPairSentenceProvider implements LabeledPairSentenceProvider {
+
+    private final List<String> sentenceL;
+    private final List<String> sentenceR;
+    private final List<String> labels;
+    private final Random rng;
+    private final int[] order;
+    private final List<String> allLabels;
+
+    private int cursor = 0;
+
+    /**
+     * Lists containing sentences to iterate over with a third for labels
+     * Sentences in the same position in the first two lists are considered a pair
+     * @param sentenceL
+     * @param sentenceR
+     * @param labelsForSentences
+     */
+    public CollectionLabeledPairSentenceProvider(@NonNull List<String> sentenceL, @NonNull List<String> sentenceR,
+                                                 @NonNull List<String> labelsForSentences) {
+        this(sentenceL, sentenceR, labelsForSentences, new Random());
+    }
+
+    /**
+     * Lists containing sentences to iterate over with a third for labels
+     * Sentences in the same position in the first two lists are considered a pair
+     * @param sentenceL
+     * @param sentenceR
+     * @param labelsForSentences
+     * @param rng If null, list order is not shuffled
+     */
+    public CollectionLabeledPairSentenceProvider(@NonNull List<String> sentenceL, List<String> sentenceR, @NonNull List<String> labelsForSentences,
+                                                 Random rng) {
+        if (sentenceR.size() != sentenceL.size()) {
+            throw new IllegalArgumentException("Sentence lists must be same size (first list size: "
+                    + sentenceL.size() + ", second list size: " + sentenceR.size() + ")");
+        }
+        if (sentenceR.size() != labelsForSentences.size()) {
+            throw new IllegalArgumentException("Sentence pairs and labels must be same size (sentence pair size: "
+                    + sentenceR.size() + ", labels size: " + labelsForSentences.size() + ")");
+        }
+
+        this.sentenceL = sentenceL;
+        this.sentenceR = sentenceR;
+        this.labels = labelsForSentences;
+        this.rng = rng;
+        if (rng == null) {
+            order = null;
+        } else {
+            order = new int[sentenceR.size()];
+            for (int i = 0; i < sentenceR.size(); i++) {
+                order[i] = i;
+            }
+
+            MathUtils.shuffleArray(order, rng);
+        }
+
+        //Collect set of unique labels for all sentences
+        Set<String> uniqueLabels = new HashSet<>(labelsForSentences);
+        allLabels = new ArrayList<>(uniqueLabels);
+        Collections.sort(allLabels);
+    }
+
+    @Override
+    public boolean hasNext() {
+        return cursor < sentenceR.size();
+    }
+
+    @Override
+    public Triple<String, String, String> nextSentencePair() {
+        Preconditions.checkState(hasNext(),"No next element available");
+        int idx;
+        if (rng == null) {
+            idx = cursor++;
+        } else {
+            idx = order[cursor++];
+        }
+        return new Triple<>(sentenceL.get(idx), sentenceR.get(idx), labels.get(idx));
+    }
+
+    @Override
+    public void reset() {
+        cursor = 0;
+        if (rng != null) {
+            MathUtils.shuffleArray(order, rng);
+        }
+    }
+
+    @Override
+    public int totalNumSentences() {
+        return sentenceR.size();
+    }
+
+    @Override
+    public List<String> allLabels() {
+        return allLabels;
+    }
+
+    @Override
+    public int numLabelClasses() {
+        return allLabels.size();
+    }
+}
+

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/provider/CollectionLabeledSentenceProvider.java

@@ -0,0 +1,112 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.iterator.provider;
+
+import lombok.NonNull;
+import org.deeplearning4j.iterator.LabeledSentenceProvider;
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.common.primitives.Pair;
+import org.nd4j.common.util.MathUtils;
+
+import java.util.*;
+
+public class CollectionLabeledSentenceProvider implements LabeledSentenceProvider {
+
+    private final List<String> sentences;
+    private final List<String> labels;
+    private final Random rng;
+    private final int[] order;
+    private final List<String> allLabels;
+
+    private int cursor = 0;
+
+    public CollectionLabeledSentenceProvider(@NonNull List<String> sentences,
+                                             @NonNull List<String> labelsForSentences) {
+        this(sentences, labelsForSentences, new Random());
+    }
+
+    public CollectionLabeledSentenceProvider(@NonNull List<String> sentences, @NonNull List<String> labelsForSentences,
+                                             Random rng) {
+        if (sentences.size() != labelsForSentences.size()) {
+            throw new IllegalArgumentException("Sentences and labels must be same size (sentences size: "
+                    + sentences.size() + ", labels size: " + labelsForSentences.size() + ")");
+        }
+
+        this.sentences = sentences;
+        this.labels = labelsForSentences;
+        this.rng = rng;
+        if (rng == null) {
+            order = null;
+        } else {
+            order = new int[sentences.size()];
+            for (int i = 0; i < sentences.size(); i++) {
+                order[i] = i;
+            }
+
+            MathUtils.shuffleArray(order, rng);
+        }
+
+        //Collect set of unique labels for all sentences
+        Set<String> uniqueLabels = new HashSet<>(labelsForSentences);
+        allLabels = new ArrayList<>(uniqueLabels);
+        Collections.sort(allLabels);
+    }
+
+    @Override
+    public boolean hasNext() {
+        return cursor < sentences.size();
+    }
+
+    @Override
+    public Pair<String, String> nextSentence() {
+        Preconditions.checkState(hasNext(), "No next element available");
+        int idx;
+        if (rng == null) {
+            idx = cursor++;
+        } else {
+            idx = order[cursor++];
+        }
+        return new Pair<>(sentences.get(idx), labels.get(idx));
+    }
+
+    @Override
+    public void reset() {
+        cursor = 0;
+        if (rng != null) {
+            MathUtils.shuffleArray(order, rng);
+        }
+    }
+
+    @Override
+    public int totalNumSentences() {
+        return sentences.size();
+    }
+
+    @Override
+    public List<String> allLabels() {
+        return allLabels;
+    }
+
+    @Override
+    public int numLabelClasses() {
+        return allLabels.size();
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/provider/FileLabeledSentenceProvider.java

@@ -0,0 +1,145 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.iterator.provider;
+
+import lombok.NonNull;
+import org.apache.commons.io.FileUtils;
+import org.deeplearning4j.iterator.LabeledSentenceProvider;
+import org.nd4j.common.collection.CompactHeapStringList;
+import org.nd4j.common.primitives.Pair;
+import org.nd4j.common.util.MathUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+
+public class FileLabeledSentenceProvider implements LabeledSentenceProvider {
+
+    private final int totalCount;
+    private final List<String> filePaths;
+    private final int[] fileLabelIndexes;
+    private final Random rng;
+    private final int[] order;
+    private final List<String> allLabels;
+
+    private int cursor = 0;
+
+    /**
+     * @param filesByLabel Key: label. Value: list of files for that label
+     */
+    public FileLabeledSentenceProvider(Map<String, List<File>> filesByLabel) {
+        this(filesByLabel, new Random());
+    }
+
+    /**
+     *
+     * @param filesByLabel Key: label. Value: list of files for that label
+     * @param rng          Random number generator. May be null.
+     */
+    public FileLabeledSentenceProvider(@NonNull Map<String, List<File>> filesByLabel, Random rng) {
+        int totalCount = 0;
+        for (List<File> l : filesByLabel.values()) {
+            totalCount += l.size();
+        }
+        this.totalCount = totalCount;
+
+        this.rng = rng;
+        if (rng == null) {
+            order = null;
+        } else {
+            order = new int[totalCount];
+            for (int i = 0; i < totalCount; i++) {
+                order[i] = i;
+            }
+
+            MathUtils.shuffleArray(order, rng);
+        }
+
+        allLabels = new ArrayList<>(filesByLabel.keySet());
+        Collections.sort(allLabels);
+
+        Map<String, Integer> labelsToIdx = new HashMap<>();
+        for (int i = 0; i < allLabels.size(); i++) {
+            labelsToIdx.put(allLabels.get(i), i);
+        }
+
+        filePaths = new CompactHeapStringList();
+        fileLabelIndexes = new int[totalCount];
+        int position = 0;
+        for (Map.Entry<String, List<File>> entry : filesByLabel.entrySet()) {
+            int labelIdx = labelsToIdx.get(entry.getKey());
+            for (File f : entry.getValue()) {
+                filePaths.add(f.getPath());
+                fileLabelIndexes[position] = labelIdx;
+                position++;
+            }
+        }
+    }
+
+    @Override
+    public boolean hasNext() {
+        return cursor < totalCount;
+    }
+
+    @Override
+    public Pair<String, String> nextSentence() {
+        int idx;
+        if (rng == null) {
+            idx = cursor++;
+        } else {
+            idx = order[cursor++];
+        }
+        File f = new File(filePaths.get(idx));
+        String label = allLabels.get(fileLabelIndexes[idx]);
+
+        String sentence;
+        try {
+            sentence = FileUtils.readFileToString(f);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+        return new Pair<>(sentence, label);
+    }
+
+    @Override
+    public void reset() {
+        cursor = 0;
+        if (rng != null) {
+            MathUtils.shuffleArray(order, rng);
+        }
+    }
+
+    @Override
+    public int totalNumSentences() {
+        return totalCount;
+    }
+
+    @Override
+    public List<String> allLabels() {
+        return allLabels;
+    }
+
+    @Override
+    public int numLabelClasses() {
+        return allLabels.size();
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/provider/LabelAwareConverter.java

@@ -0,0 +1,72 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.iterator.provider;
+
+import lombok.NonNull;
+import org.deeplearning4j.iterator.LabeledSentenceProvider;
+import org.deeplearning4j.text.documentiterator.LabelAwareIterator;
+import org.deeplearning4j.text.documentiterator.LabelledDocument;
+import org.nd4j.common.primitives.Pair;
+
+import java.util.List;
+
+public class LabelAwareConverter implements LabeledSentenceProvider {
+    private LabelAwareIterator backingIterator;
+    private List<String> labels;
+
+    public LabelAwareConverter(@NonNull LabelAwareIterator iterator, @NonNull List<String> labels) {
+        this.backingIterator = iterator;
+        this.labels = labels;
+    }
+
+    @Override
+    public boolean hasNext() {
+        return backingIterator.hasNext();
+    }
+
+    @Override
+    public Pair<String, String> nextSentence() {
+        LabelledDocument document = backingIterator.nextDocument();
+
+        // TODO: probably worth to allow more then one label? i.e. pass same document twice, sequentially
+        return Pair.makePair(document.getContent(), document.getLabels().get(0));
+    }
+
+    @Override
+    public void reset() {
+        backingIterator.reset();
+    }
+
+    @Override
+    public int totalNumSentences() {
+        return -1;
+    }
+
+    @Override
+    public List<String> allLabels() {
+        return labels;
+    }
+
+    @Override
+    public int numLabelClasses() {
+        return labels.size();
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/WeightLookupTable.java

@@ -0,0 +1,143 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings;
+
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.deeplearning4j.core.ui.UiConnectionInfo;
+import org.nd4j.linalg.api.ndarray.INDArray;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * General weight lookup table
+ *
+ * @author Adam Gibson
+ */
+public interface WeightLookupTable<T extends SequenceElement> extends Serializable {
+
+    /**
+     * Returns unique ID of this table.
+     * Used for JointStorage/DistributedLookupTable mechanics
+     *
+     * @return ID of this table
+     */
+    Long getTableId();
+
+    /**
+     * Set's table Id.
+     * Please note, it should be unique withing Joint/Distributed LookupTable
+     *
+     * @param tableId
+     */
+    void setTableId(Long tableId);
+
+    /**
+     * The layer size for the lookup table
+     * @return the layer size for the lookup table
+     */
+    int layerSize();
+
+
+
+    /**
+     * Clear out all weights regardless
+     * @param reset
+     */
+    void resetWeights(boolean reset);
+
+
+
+    /**
+     *
+     * @param codeIndex
+     * @param code
+     */
+    void putCode(int codeIndex, INDArray code);
+
+    /**
+     * Loads the co-occurrences for the given codes
+     * @param codes the codes to load
+     * @return an ndarray of code.length by layerSize
+     */
+    INDArray loadCodes(int[] codes);
+
+    /**
+     * Iterate on the given 2 vocab words
+     * @param w1 the first word to iterate on
+     * @param w2 the second word to iterate on
+     */
+    @Deprecated
+    void iterate(T w1, T w2);
+
+    /**
+     * Iterate on the given 2 vocab words
+     * @param w1 the first word to iterate on
+     * @param w2 the second word to iterate on
+     * @param nextRandom nextRandom for sampling
+     * @param alpha the alpha to use for learning
+     */
+    @Deprecated
+    void iterateSample(T w1, T w2, AtomicLong nextRandom, double alpha);
+
+
+    /**
+     * Inserts a word vector
+     * @param word the word to insert
+     * @param vector the vector to insert
+     */
+    void putVector(String word, INDArray vector);
+
+    /**
+     *
+     * @param word
+     * @return
+     */
+    INDArray vector(String word);
+
+    /**
+     * Reset the weights of the cache
+     */
+    void resetWeights();
+
+
+    /**
+     * Sets the learning rate
+     * @param lr
+     */
+    void setLearningRate(double lr);
+
+    /**
+     * Iterates through all of the vectors in the cache
+     * @return an iterator for all vectors in the cache
+     */
+    Iterator<INDArray> vectors();
+
+    INDArray getWeights();
+
+    /**
+     * Returns corresponding vocabulary
+     */
+    VocabCache<T> getVocabCache();
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/inmemory/InMemoryLookupTable.java

@@ -0,0 +1,633 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings.inmemory;
+
+import org.nd4j.shade.guava.util.concurrent.AtomicDouble;
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+import org.apache.commons.math3.util.FastMath;
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.Word2Vec;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.linalg.api.buffer.DataType;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.api.rng.Random;
+import org.nd4j.linalg.exception.ND4JIllegalStateException;
+import org.nd4j.linalg.factory.Nd4j;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+
+/**
+ * Default word lookup table
+ *
+ * @author Adam Gibson
+ */
+public class InMemoryLookupTable<T extends SequenceElement> implements WeightLookupTable<T> {
+
+    private static final Logger log = LoggerFactory.getLogger(InMemoryLookupTable.class);
+
+    protected INDArray syn0, syn1;
+    protected int vectorLength;
+    protected transient Random rng = Nd4j.getRandom();
+    protected AtomicDouble lr = new AtomicDouble(25e-3);
+    protected double[] expTable;
+    protected static double MAX_EXP = 6;
+    protected long seed = 123;
+    //negative sampling table
+    protected INDArray table, syn1Neg;
+    protected boolean useAdaGrad;
+    protected double negative = 0;
+    protected boolean useHS = true;
+    protected VocabCache<T> vocab;
+    protected Map<Integer, INDArray> codes = new ConcurrentHashMap<>();
+
+
+
+
+    @Getter
+    @Setter
+    protected Long tableId;
+
+    public InMemoryLookupTable() {}
+
+    public InMemoryLookupTable(VocabCache<T> vocab, int vectorLength, boolean useAdaGrad, double lr, Random gen,
+                               double negative, boolean useHS) {
+        this(vocab, vectorLength, useAdaGrad, lr, gen, negative);
+        this.useHS = useHS;
+    }
+
+    public InMemoryLookupTable(VocabCache<T> vocab, int vectorLength, boolean useAdaGrad, double lr, Random gen,
+                               double negative) {
+        this.vocab = vocab;
+        this.vectorLength = vectorLength;
+        this.useAdaGrad = useAdaGrad;
+        this.lr.set(lr);
+        this.rng = gen;
+        this.negative = negative;
+        initExpTable();
+
+    }
+
+
+
+    public double[] getExpTable() {
+        if(expTable == null)
+            initExpTable();
+        return expTable;
+    }
+
+    public void setExpTable(double[] expTable) {
+        this.expTable = expTable;
+    }
+
+
+    @Override
+    public int layerSize() {
+        return vectorLength;
+    }
+
+    @Override
+    public void resetWeights(boolean reset) {
+        if (this.rng == null)
+            this.rng = Nd4j.getRandom();
+
+        this.rng.setSeed(seed);
+
+        if (syn0 == null || reset) {
+            syn0 = Nd4j.rand(new int[] {vocab.numWords(), vectorLength}, rng).subi(0.5).divi(vectorLength);
+        }
+
+        if ((syn1 == null || reset) && useHS) {
+            log.info("Initializing syn1...");
+            syn1 = syn0.like();
+        }
+
+        initNegative();
+    }
+
+
+
+    /**
+     * @param codeIndex
+     * @param code
+     */
+    @Override
+    public void putCode(int codeIndex, INDArray code) {
+        codes.put(codeIndex, code);
+    }
+
+    /**
+     * Loads the co-occurrences for the given codes
+     *
+     * @param codes the codes to load
+     * @return an ndarray of code.length by layerSize
+     */
+    @Override
+    public INDArray loadCodes(int[] codes) {
+        return syn1.getRows(codes);
+    }
+
+
+    public synchronized void initNegative() {
+        if (negative > 0 && syn1Neg == null) {
+            syn1Neg = Nd4j.zeros(syn0.shape());
+            makeTable(Math.max(expTable.length, 100000), 0.75);
+        }
+    }
+
+
+    protected void initExpTable() {
+        expTable = new double[100000];
+        for (int i = 0; i < expTable.length; i++) {
+            double tmp = FastMath.exp((i / (double) expTable.length * 2 - 1) * MAX_EXP);
+            expTable[i] = tmp / (tmp + 1.0);
+        }
+    }
+
+
+
+    /**
+     * Iterate on the given 2 vocab words
+     *
+     * @param w1 the first word to iterate on
+     * @param w2 the second word to iterate on
+     * @param nextRandom next random for sampling
+     */
+    @Override
+    @Deprecated
+    public void iterateSample(T w1, T w2, AtomicLong nextRandom, double alpha) {
+        if (w2 == null || w2.getIndex() < 0 || w1.getIndex() == w2.getIndex() || w1.getLabel().equals("STOP")
+                || w2.getLabel().equals("STOP") || w1.getLabel().equals("UNK") || w2.getLabel().equals("UNK"))
+            return;
+        //current word vector
+        INDArray l1 = this.syn0.slice(w2.getIndex());
+
+
+        //error for current word and context
+        INDArray neu1e = Nd4j.create(vectorLength);
+
+
+        for (int i = 0; i < w1.getCodeLength(); i++) {
+            int code = w1.getCodes().get(i);
+            int point = w1.getPoints().get(i);
+            if (point >= syn0.rows() || point < 0)
+                throw new IllegalStateException("Illegal point " + point);
+            //other word vector
+
+            INDArray syn1 = this.syn1.slice(point);
+
+
+            double dot = Nd4j.getBlasWrapper().dot(l1, syn1);
+
+            if (dot < -MAX_EXP || dot >= MAX_EXP)
+                continue;
+
+
+            int idx = (int) ((dot + MAX_EXP) * ((double) expTable.length / MAX_EXP / 2.0));
+            if (idx >= expTable.length)
+                continue;
+
+            //score
+            double f = expTable[idx];
+            //gradient
+            double g = useAdaGrad ? w1.getGradient(i, (1 - code - f), lr.get()) : (1 - code - f) * alpha;
+
+            Nd4j.getBlasWrapper().level1().axpy(syn1.length(), g, syn1, neu1e);
+            Nd4j.getBlasWrapper().level1().axpy(syn1.length(), g, l1, syn1);
+
+        }
+
+
+        int target = w1.getIndex();
+        int label;
+        //negative sampling
+        if (negative > 0)
+            for (int d = 0; d < negative + 1; d++) {
+                if (d == 0)
+                    label = 1;
+                else {
+                    nextRandom.set(Math.abs(nextRandom.get() * 25214903917L + 11));
+
+                    int idx = (int) Math.abs((int) (nextRandom.get() >> 16) % table.length());
+
+                    target = table.getInt(idx);
+                    if (target <= 0)
+                        target = (int) nextRandom.get() % (vocab.numWords() - 1) + 1;
+
+                    if (target == w1.getIndex())
+                        continue;
+                    label = 0;
+                }
+
+
+                if (target >= syn1Neg.rows() || target < 0)
+                    continue;
+
+                double f = Nd4j.getBlasWrapper().dot(l1, syn1Neg.slice(target));
+                double g;
+                if (f > MAX_EXP)
+                    g = useAdaGrad ? w1.getGradient(target, (label - 1), alpha) : (label - 1) * alpha;
+                else if (f < -MAX_EXP)
+                    g = label * (useAdaGrad ? w1.getGradient(target, alpha, alpha) : alpha);
+                else
+                    g = useAdaGrad ? w1
+                            .getGradient(target,
+                                    label - expTable[(int) ((f + MAX_EXP)
+                                            * (expTable.length / MAX_EXP / 2))],
+                                    alpha)
+                            : (label - expTable[(int) ((f + MAX_EXP) * (expTable.length / MAX_EXP / 2))])
+                            * alpha;
+                if (syn0.data().dataType() == DataType.DOUBLE)
+                    Nd4j.getBlasWrapper().axpy(g, syn1Neg.slice(target), neu1e);
+                else
+                    Nd4j.getBlasWrapper().axpy((float) g, syn1Neg.slice(target), neu1e);
+
+                if (syn0.data().dataType() == DataType.DOUBLE)
+                    Nd4j.getBlasWrapper().axpy(g, l1, syn1Neg.slice(target));
+                else
+                    Nd4j.getBlasWrapper().axpy((float) g, l1, syn1Neg.slice(target));
+            }
+
+        if (syn0.data().dataType() == DataType.DOUBLE)
+            Nd4j.getBlasWrapper().axpy(1.0, neu1e, l1);
+
+        else
+            Nd4j.getBlasWrapper().axpy(1.0f, neu1e, l1);
+
+    }
+
+    public boolean isUseAdaGrad() {
+        return useAdaGrad;
+    }
+
+    public void setUseAdaGrad(boolean useAdaGrad) {
+        this.useAdaGrad = useAdaGrad;
+    }
+
+    public double getNegative() {
+        return negative;
+    }
+
+    public void setUseHS(boolean useHS) {
+        this.useHS = useHS;
+    }
+
+    public void setNegative(double negative) {
+        this.negative = negative;
+    }
+
+    /**
+     * Iterate on the given 2 vocab words
+     *
+     * @param w1 the first word to iterate on
+     * @param w2 the second word to iterate on
+     */
+    @Deprecated
+    @Override
+    public void iterate(T w1, T w2) {
+
+    }
+
+
+    /**
+     * Reset the weights of the cache
+     */
+    @Override
+    public void resetWeights() {
+        resetWeights(true);
+    }
+
+
+    protected void makeTable(int tableSize, double power) {
+        int vocabSize = syn0.rows();
+        table = Nd4j.create(tableSize);
+        double trainWordsPow = 0.0;
+        for (String word : vocab.words()) {
+            trainWordsPow += Math.pow(vocab.wordFrequency(word), power);
+        }
+
+        int wordIdx = 0;
+        String word = vocab.wordAtIndex(wordIdx);
+        double d1 = Math.pow(vocab.wordFrequency(word), power) / trainWordsPow;
+        for (int i = 0; i < tableSize; i++) {
+            table.putScalar(i, wordIdx);
+            double mul = i * 1.0 / (double) tableSize;
+            if (mul > d1) {
+                if (wordIdx < vocabSize - 1)
+                    wordIdx++;
+                word = vocab.wordAtIndex(wordIdx);
+                String wordAtIndex = vocab.wordAtIndex(wordIdx);
+                if (word == null)
+                    continue;
+                d1 += Math.pow(vocab.wordFrequency(wordAtIndex), power) / trainWordsPow;
+            }
+        }
+    }
+
+    /**
+     * Inserts a word vector
+     *
+     * @param word   the word to insert
+     * @param vector the vector to insert
+     */
+    @Override
+    public void putVector(String word, INDArray vector) {
+        if (word == null)
+            throw new IllegalArgumentException("No null words allowed");
+        if (vector == null)
+            throw new IllegalArgumentException("No null vectors allowed");
+        int idx = vocab.indexOf(word);
+        syn0.slice(idx).assign(vector);
+
+    }
+
+    public INDArray getTable() {
+        return table;
+    }
+
+    public void setTable(INDArray table) {
+        this.table = table;
+    }
+
+    public INDArray getSyn1Neg() {
+        return syn1Neg;
+    }
+
+    public void setSyn1Neg(INDArray syn1Neg) {
+        this.syn1Neg = syn1Neg;
+    }
+
+    /**
+     * @param word
+     * @return
+     */
+    @Override
+    public INDArray vector(String word) {
+        if (word == null)
+            return null;
+        int idx = vocab.indexOf(word);
+        if (idx < 0) {
+            idx = vocab.indexOf(Word2Vec.DEFAULT_UNK);
+            if (idx < 0)
+                return null;
+        }
+        return syn0.getRow(idx, false);
+    }
+
+    @Override
+    public void setLearningRate(double lr) {
+        this.lr.set(lr);
+    }
+
+    @Override
+    public Iterator<INDArray> vectors() {
+        return new WeightIterator();
+    }
+
+    @Override
+    public INDArray getWeights() {
+        return syn0;
+    }
+
+
+    protected class WeightIterator implements Iterator<INDArray> {
+        protected int currIndex = 0;
+
+        @Override
+        public boolean hasNext() {
+            return currIndex < syn0.rows();
+        }
+
+        @Override
+        public INDArray next() {
+            INDArray ret = syn0.slice(currIndex);
+            currIndex++;
+            return ret;
+        }
+
+        @Override
+        public void remove() {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+    public INDArray getSyn0() {
+        return syn0;
+    }
+
+    public void setSyn0(@NonNull INDArray syn0) {
+        Preconditions.checkArgument(!syn0.isEmpty(), "syn0 can't be empty");
+        Preconditions.checkArgument(syn0.rank() == 2, "syn0 must have rank 2");
+
+        this.syn0 = syn0;
+        this.vectorLength = syn0.columns();
+    }
+
+    public INDArray getSyn1() {
+        return syn1;
+    }
+
+    public void setSyn1(@NonNull INDArray syn1) {
+        this.syn1 = syn1;
+    }
+
+    @Override
+    public VocabCache<T> getVocabCache() {
+        return vocab;
+    }
+
+    public void setVectorLength(int vectorLength) {
+        this.vectorLength = vectorLength;
+    }
+
+    /**
+     * This method is deprecated, since all logic was pulled out from this class and is not used anymore.
+     * However this method will be around for a while, due to backward compatibility issues.
+     * @return initial learning rate
+     */
+    @Deprecated
+    public AtomicDouble getLr() {
+        return lr;
+    }
+
+    public void setLr(AtomicDouble lr) {
+        this.lr = lr;
+    }
+
+    public VocabCache getVocab() {
+        return vocab;
+    }
+
+    public void setVocab(VocabCache vocab) {
+        this.vocab = vocab;
+    }
+
+    public Map<Integer, INDArray> getCodes() {
+        return codes;
+    }
+
+    public void setCodes(Map<Integer, INDArray> codes) {
+        this.codes = codes;
+    }
+
+    public static class Builder<T extends SequenceElement> {
+        protected int vectorLength = 100;
+        protected boolean useAdaGrad = false;
+        protected double lr = 0.025;
+        protected Random gen = Nd4j.getRandom();
+        protected long seed = 123;
+        protected double negative = 0;
+        protected VocabCache<T> vocabCache;
+        protected boolean useHS = true;
+
+
+
+        public Builder<T> useHierarchicSoftmax(boolean reallyUse) {
+            this.useHS = reallyUse;
+            return this;
+        }
+
+        public Builder<T> cache(@NonNull VocabCache<T> vocab) {
+            this.vocabCache = vocab;
+            return this;
+        }
+
+        public Builder<T> negative(double negative) {
+            this.negative = negative;
+            return this;
+        }
+
+        public Builder<T> vectorLength(int vectorLength) {
+            this.vectorLength = vectorLength;
+            return this;
+        }
+
+        public Builder<T> useAdaGrad(boolean useAdaGrad) {
+            this.useAdaGrad = useAdaGrad;
+            return this;
+        }
+
+        /**
+         * This method is deprecated, since all logic was pulled out from this class
+         * @param lr
+         * @return
+         */
+        @Deprecated
+        public Builder<T> lr(double lr) {
+            this.lr = lr;
+            return this;
+        }
+
+        public Builder<T> gen(Random gen) {
+            this.gen = gen;
+            return this;
+        }
+
+        public Builder<T> seed(long seed) {
+            this.seed = seed;
+            return this;
+        }
+
+
+
+        public InMemoryLookupTable<T> build() {
+            if (vocabCache == null)
+                throw new IllegalStateException("Vocab cache must be specified");
+
+            InMemoryLookupTable<T> table =
+                    new InMemoryLookupTable<>(vocabCache, vectorLength, useAdaGrad, lr, gen, negative, useHS);
+            table.seed = seed;
+
+            return table;
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "InMemoryLookupTable{" + "syn0=" + syn0 + ", syn1=" + syn1 + ", vectorLength=" + vectorLength + ", rng="
+                + rng + ", lr=" + lr + ", expTable=" + Arrays.toString(expTable) + ", seed=" + seed + ", table="
+                + table + ", syn1Neg=" + syn1Neg + ", useAdaGrad=" + useAdaGrad + ", negative=" + negative
+                + ", vocab=" + vocab + ", codes=" + codes + '}';
+    }
+
+    /**
+     * This method consumes weights of a given InMemoryLookupTable
+     *
+     * PLEASE NOTE: this method explicitly resets current weights
+     *
+     * @param srcTable
+     */
+    public void consume(InMemoryLookupTable<T> srcTable) {
+        if (srcTable.vectorLength != this.vectorLength)
+            throw new IllegalStateException("You can't consume lookupTable with different vector lengths");
+
+        if (srcTable.syn0 == null)
+            throw new IllegalStateException("Source lookupTable Syn0 is NULL");
+
+        this.resetWeights(true);
+
+        AtomicInteger cntHs = new AtomicInteger(0);
+        AtomicInteger cntNg = new AtomicInteger(0);
+
+        if (srcTable.syn0.rows() > this.syn0.rows())
+            throw new IllegalStateException(
+                    "You can't consume lookupTable with built for larger vocabulary without updating your vocabulary first");
+
+        for (int x = 0; x < srcTable.syn0.rows(); x++) {
+            this.syn0.putRow(x, srcTable.syn0.getRow(x));
+
+            if (this.syn1 != null && srcTable.syn1 != null)
+                this.syn1.putRow(x, srcTable.syn1.getRow(x));
+            else if (cntHs.incrementAndGet() == 1)
+                log.info("Skipping syn1 merge");
+
+            if (this.syn1Neg != null && srcTable.syn1Neg != null) {
+                this.syn1Neg.putRow(x, srcTable.syn1Neg.getRow(x));
+            } else if (cntNg.incrementAndGet() == 1)
+                log.info("Skipping syn1Neg merge");
+
+            if (cntHs.get() > 0 && cntNg.get() > 0)
+                throw new ND4JIllegalStateException("srcTable has no syn1/syn1neg");
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof InMemoryLookupTable)) return false;
+        InMemoryLookupTable<?> that = (InMemoryLookupTable<?>) o;
+        return vectorLength == that.vectorLength && isUseAdaGrad() == that.isUseAdaGrad() && Double.compare(that.getNegative(), getNegative()) == 0 && useHS == that.useHS && Objects.equals(getSyn0(), that.getSyn0()) && Objects.equals(getSyn1(), that.getSyn1()) && Objects.equals(rng, that.rng) && Objects.equals(getTable(), that.getTable()) && Objects.equals(getSyn1Neg(), that.getSyn1Neg()) && Objects.equals(getVocab(), that.getVocab()) && Objects.equals(getCodes(), that.getCodes()) && Objects.equals(getTableId(), that.getTableId());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getSyn0(), getSyn1(), vectorLength, rng, getTable(), getSyn1Neg(), isUseAdaGrad(), getNegative(), useHS, getVocab(), getCodes(), getTableId());
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/ElementsLearningAlgorithm.java

@@ -0,0 +1,70 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings.learning;
+
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.embeddings.learning.impl.elements.BatchSequences;
+import org.deeplearning4j.models.embeddings.loader.VectorsConfiguration;
+import org.deeplearning4j.models.sequencevectors.interfaces.SequenceIterator;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.nd4j.linalg.api.memory.conf.WorkspaceConfiguration;
+import org.nd4j.linalg.api.memory.enums.*;
+import org.nd4j.linalg.api.ndarray.INDArray;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+public interface ElementsLearningAlgorithm<T extends SequenceElement> {
+    default WorkspaceConfiguration workspaceConfig() {
+       return  WorkspaceConfiguration.builder()
+               .policyAllocation(AllocationPolicy.STRICT)
+               .maxSize(3000)
+               .minSize(1000)
+               .policyReset(ResetPolicy.BLOCK_LEFT)
+               .policySpill(SpillPolicy.EXTERNAL)
+               .initialSize(1000)
+               .build();
+    }
+
+    String getCodeName();
+
+    void configure(VocabCache<T> vocabCache, WeightLookupTable<T> lookupTable, VectorsConfiguration configuration);
+
+    void pretrain(SequenceIterator<T> iterator);
+
+    /**
+     * This method does training over the sequence of elements passed into it
+     *
+     * @param sequence
+     * @param nextRandom
+     * @param learningRate
+     * @return average score for this sequence
+     */
+    double learnSequence(Sequence<T> sequence, AtomicLong nextRandom, double learningRate);
+
+
+    boolean isEarlyTerminationHit();
+
+    void finish();
+
+    void finish(INDArray inferenceVector);
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/SequenceLearningAlgorithm.java

@@ -0,0 +1,73 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings.learning;
+
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.embeddings.learning.impl.elements.BatchSequences;
+import org.deeplearning4j.models.embeddings.loader.VectorsConfiguration;
+import org.deeplearning4j.models.sequencevectors.interfaces.SequenceIterator;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.nd4j.linalg.api.ndarray.INDArray;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+public interface SequenceLearningAlgorithm<T extends SequenceElement> {
+
+    String getCodeName();
+
+    void configure(VocabCache<T> vocabCache, WeightLookupTable<T> lookupTable, VectorsConfiguration configuration);
+
+    void pretrain(SequenceIterator<T> iterator);
+
+    /**
+     * This method does training over the sequence of elements passed into it
+     *
+     * @param sequence
+     * @param nextRandom
+     * @param learningRate
+     * @return average score for this sequence
+     */
+    double learnSequence(Sequence<T> sequence, AtomicLong nextRandom, double learningRate);
+
+    boolean isEarlyTerminationHit();
+
+    INDArray inferSequence(INDArray inferenceVector, Sequence<T> sequence, long nextRandom, double learningRate, double minLearningRate,
+                           int iterations);
+
+    /**
+     * This method does training on previously unseen paragraph, and returns inferred vector
+     *
+     * @param sequence
+     * @param nextRandom
+     * @param learningRate
+     * @return
+     */
+    INDArray inferSequence(Sequence<T> sequence, long nextRandom, double learningRate, double minLearningRate,
+                    int iterations);
+
+    ElementsLearningAlgorithm<T> getElementsLearningAlgorithm();
+
+    void finish();
+
+    void finish(INDArray inferenceVector);
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/BatchItem.java

@@ -0,0 +1,147 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings.learning.impl.elements;
+
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.nd4j.common.primitives.CounterMap;
+import org.nd4j.shade.guava.collect.HashBasedTable;
+import org.nd4j.shade.guava.collect.Table;
+
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class BatchItem<T extends SequenceElement>  {
+    private T word;
+    private T lastWord;
+
+
+
+    private int[] windowWords; // CBOW only
+    private boolean[] wordStatuses;
+
+    private long randomValue;
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        BatchItem<?> batchItem = (BatchItem<?>) o;
+        return randomValue == batchItem.randomValue && Double.compare(batchItem.alpha, alpha) == 0 && windowWordsLength == batchItem.windowWordsLength && numLabel == batchItem.numLabel && Objects.equals(word, batchItem.word) && Objects.equals(lastWord, batchItem.lastWord) && Arrays.equals(windowWords, batchItem.windowWords) && Arrays.equals(wordStatuses, batchItem.wordStatuses);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Objects.hash(word, lastWord, randomValue, alpha, windowWordsLength, numLabel);
+        result = 31 * result + Arrays.hashCode(windowWords);
+        result = 31 * result + Arrays.hashCode(wordStatuses);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "BatchItem{" +
+                "word=" + word +
+                ", lastWord=" + lastWord +
+                ", windowWords=" + Arrays.toString(windowWords) +
+                ", wordStatuses=" + Arrays.toString(wordStatuses) +
+                ", randomValue=" + randomValue +
+                ", alpha=" + alpha +
+                ", windowWordsLength=" + windowWordsLength +
+                ", numLabel=" + numLabel +
+                '}';
+    }
+
+    private double alpha;
+    private int windowWordsLength;
+
+    private int numLabel;
+
+
+
+
+    public BatchItem(T word, T lastWord, long randomValue, double alpha) {
+        this.word = word;
+        this.lastWord = lastWord;
+        this.randomValue = randomValue;
+        this.alpha = alpha;
+    }
+
+    public BatchItem(T word, int[] windowWords, boolean[] wordStatuses, long randomValue, double alpha, int numLabel) {
+        this.word = word;
+        this.lastWord = lastWord;
+        this.randomValue = randomValue;
+        this.alpha = alpha;
+        this.numLabel = numLabel;
+        this.windowWords = windowWords.clone();
+        this.wordStatuses = wordStatuses.clone();
+
+    }
+
+    public BatchItem(T word, int[] windowWords, boolean[] wordStatuses, long randomValue, double alpha) {
+        this.word = word;
+        this.lastWord = lastWord;
+        this.randomValue = randomValue;
+        this.alpha = alpha;
+        this.windowWords = windowWords.clone();
+        this.wordStatuses = wordStatuses.clone();
+    }
+
+    public T getWord() {
+        return word;
+    }
+
+    public void setWord(T word) {
+        this.word = word;
+    }
+
+    public T getLastWord() {
+        return lastWord;
+    }
+
+    public void setLastWord(T lastWord) {
+        this.lastWord = lastWord;
+    }
+
+    public long getRandomValue() {
+        return randomValue;
+    }
+
+    public double getAlpha() {
+        return alpha;
+    }
+
+    public void setAlpha(double alpha) {
+        this.alpha = alpha;
+    }
+
+    public int[] getWindowWords() {
+        return windowWords;
+    }
+
+    public boolean[] getWordStatuses() {
+        return wordStatuses;
+    }
+
+    public int getNumLabel() {
+        return numLabel;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/BatchSequences.java

@@ -0,0 +1,75 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings.learning.impl.elements;
+
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Slf4j
+public class BatchSequences<T extends SequenceElement> {
+
+    private int batches;
+
+    List<BatchItem<T>> buffer = new ArrayList<>();
+
+    public BatchSequences(int batches) {
+        this.batches = batches;
+    }
+
+    public void put(T word, T lastWord, long randomValue, double alpha) {
+        BatchItem<T> newItem = new BatchItem<>(word, lastWord, randomValue, alpha);
+        buffer.add(newItem);
+    }
+
+    public void put(T word, int[] windowWords, boolean[] wordStatuses, long randomValue, double alpha) {
+        BatchItem<T> newItem = new BatchItem<>(word, windowWords, wordStatuses, randomValue, alpha);
+        buffer.add(newItem);
+    }
+
+    public void put(T word, int[] windowWords, boolean[] wordStatuses, long randomValue, double alpha, int numLabels) {
+        BatchItem<T> newItem = new BatchItem<>(word, windowWords, wordStatuses, randomValue, alpha, numLabels);
+        buffer.add(newItem);
+    }
+
+    public List<BatchItem<T>> get(int chunkNo) {
+        List<BatchItem<T>> retVal = new ArrayList<>();
+
+        for (int i = 0 + chunkNo * batches; (i < batches + chunkNo * batches) && (i < buffer.size()); ++i) {
+            BatchItem<T> value = buffer.get(i);
+            retVal.add(value);
+        }
+        return retVal;
+    }
+
+    public int size() {
+        return buffer.size();
+    }
+
+    public void clear() {
+        buffer.clear();
+    }
+
+
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/CBOW.java

@@ -0,0 +1,523 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings.learning.impl.elements;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import lombok.Setter;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.deeplearning4j.config.DL4JSystemProperties;
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable;
+import org.deeplearning4j.models.embeddings.learning.ElementsLearningAlgorithm;
+import org.deeplearning4j.models.embeddings.loader.VectorsConfiguration;
+import org.deeplearning4j.models.sequencevectors.interfaces.SequenceIterator;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.nd4j.linalg.api.buffer.DataType;
+import org.nd4j.linalg.api.memory.MemoryWorkspace;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.api.ops.impl.nlp.CbowInference;
+import org.nd4j.linalg.api.ops.impl.nlp.CbowRound;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.util.DeviceLocalNDArray;
+import org.nd4j.shade.guava.cache.Cache;
+import org.nd4j.shade.guava.cache.CacheBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class CBOW<T extends SequenceElement> implements ElementsLearningAlgorithm<T> {
+    private VocabCache<T> vocabCache;
+    private WeightLookupTable<T> lookupTable;
+    private VectorsConfiguration configuration;
+
+    private static final Logger logger = LoggerFactory.getLogger(CBOW.class);
+
+
+    protected int window;
+    protected boolean useAdaGrad;
+    protected double negative;
+    protected double sampling;
+    protected int[] variableWindows;
+    protected int workers = Runtime.getRuntime().availableProcessors();
+    private Cache<IterationArraysKey, Queue<IterationArrays>> iterationArrays = CacheBuilder.newBuilder()
+            .maximumSize(Integer.parseInt(System.getProperty(DL4JSystemProperties.NLP_CACHE_SIZE,"10000")))
+            .build();
+
+    protected int maxQueueSize = Integer.parseInt(System.getProperty(DL4JSystemProperties.NLP_QUEUE_SIZE,"1000"));
+
+
+
+    public int getWorkers() {
+        return workers;
+    }
+
+    public void setWorkers(int workers) {
+        this.workers = workers;
+    }
+
+    @Getter
+    @Setter
+    protected DeviceLocalNDArray syn0, syn1, syn1Neg, expTable, table;
+
+    protected ThreadLocal<List<BatchItem<T>>> batches = new ThreadLocal<>();
+
+    public List<BatchItem<T>> getBatch() {
+        if(batches.get() == null)
+            batches.set(new ArrayList<>());
+        return batches.get();
+    }
+
+
+    public void addBatchItem(BatchItem<T> batchItem) {
+        getBatch().add(batchItem);
+    }
+
+
+    public void clearBatch() {
+        getBatch().clear();
+    }
+
+    @Override
+    public String getCodeName() {
+        return "CBOW";
+    }
+
+    @Override
+    public void configure(@NonNull VocabCache<T> vocabCache, @NonNull WeightLookupTable<T> lookupTable,
+                          @NonNull VectorsConfiguration configuration) {
+        this.vocabCache = vocabCache;
+        this.lookupTable = lookupTable;
+        this.configuration = configuration;
+
+        this.window = configuration.getWindow();
+        this.useAdaGrad = configuration.isUseAdaGrad();
+        this.negative = configuration.getNegative();
+        this.sampling = configuration.getSampling();
+        this.workers = configuration.getWorkers();
+        if (configuration.getNegative() > 0) {
+            if (((InMemoryLookupTable<T>) lookupTable).getSyn1Neg() == null) {
+                logger.info("Initializing syn1Neg...");
+                ((InMemoryLookupTable<T>) lookupTable).setUseHS(configuration.isUseHierarchicSoftmax());
+                ((InMemoryLookupTable<T>) lookupTable).setNegative(configuration.getNegative());
+                lookupTable.resetWeights(false);
+            }
+        }
+
+
+        this.syn0 = new DeviceLocalNDArray(((InMemoryLookupTable<T>) lookupTable).getSyn0());
+        this.syn1 = new DeviceLocalNDArray(((InMemoryLookupTable<T>) lookupTable).getSyn1());
+        this.syn1Neg = new DeviceLocalNDArray(((InMemoryLookupTable<T>) lookupTable).getSyn1Neg());
+        this.expTable = new DeviceLocalNDArray(Nd4j.create(((InMemoryLookupTable<T>) lookupTable).getExpTable(),
+                new long[]{((InMemoryLookupTable<T>) lookupTable).getExpTable().length}, syn0.get() == null ? DataType.DOUBLE :  syn0.get().dataType()));
+        this.table = new DeviceLocalNDArray(((InMemoryLookupTable<T>) lookupTable).getTable());
+        this.variableWindows = configuration.getVariableWindows();
+    }
+
+    /**
+     * CBOW doesn't involve any pretraining
+     *
+     * @param iterator
+     */
+    @Override
+    public void pretrain(SequenceIterator<T> iterator) {
+        // no-op
+    }
+
+    @Override
+    public void finish() {
+        if (batches != null && batches.get() != null && !batches.get().isEmpty()) {
+            doExec(batches.get(),null);
+            batches.get().clear();
+        }
+    }
+
+    @Override
+    public void finish(INDArray inferenceVector) {
+        if (batches != null && batches.get() != null && !batches.get().isEmpty()) {
+            doExec(batches.get(),inferenceVector);
+            batches.get().clear();
+        }
+    }
+
+
+    @Override
+    public double learnSequence(Sequence<T> sequence, AtomicLong nextRandom, double learningRate) {
+        Sequence<T> tempSequence = sequence;
+        if (sampling > 0)
+            tempSequence = applySubsampling(sequence, nextRandom);
+
+        int currentWindow = window;
+
+        if (variableWindows != null && variableWindows.length != 0) {
+            currentWindow = variableWindows[RandomUtils.nextInt(0, variableWindows.length)];
+        }
+
+        for (int i = 0; i < tempSequence.getElements().size(); i++) {
+            nextRandom.set(Math.abs(nextRandom.get() * 25214903917L + 11));
+            cbow(i, tempSequence.getElements(), (int) nextRandom.get() % currentWindow, nextRandom, learningRate,
+                    currentWindow, null);
+        }
+
+        if (getBatch() != null && getBatch().size() >= configuration.getBatchSize()) {
+            doExec(getBatch(),null);
+            getBatch().clear();
+        }
+
+
+        return 0;
+    }
+
+    @Override
+    public boolean isEarlyTerminationHit() {
+        return false;
+    }
+
+    @Data
+    @AllArgsConstructor
+    @Builder
+    @NoArgsConstructor
+    public static class IterationArraysKey {
+        private int itemSize;
+        private int maxCols;
+    }
+
+
+    public double doExec(List<BatchItem<T>> items, INDArray inferenceVector) {
+        try(MemoryWorkspace workspace = Nd4j.getWorkspaceManager().scopeOutOfWorkspaces()) {
+            boolean useHS = configuration.isUseHierarchicSoftmax();
+            boolean useNegative = configuration.getNegative() > 0;
+            boolean useInference = inferenceVector != null;
+            if (items.size() > 1) {
+                int maxCols = 1;
+                for (int i = 0; i < items.size(); i++) {
+                    int curr = items.get(i).getWord().getCodeLength();
+                    if (curr > maxCols)
+                        maxCols = curr;
+                }
+
+
+                boolean hasNumLabels = false;
+
+                int maxWinWordsCols = -1;
+                for (int i = 0; i < items.size(); ++i) {
+                    int curr = items.get(i).getWord().getCodeLength();
+                    if (curr > maxWinWordsCols)
+                        maxWinWordsCols = curr;
+                }
+
+
+
+
+
+                INDArray inputWindowWords;
+                INDArray inputWordsStatuses;
+                INDArray randoms;
+                INDArray alphas;
+                INDArray currentWindowIndexes;
+                INDArray codes;
+                INDArray indices;
+                INDArray numLabelsArray;
+
+
+                IterationArraysKey key = IterationArraysKey.builder()
+                        .itemSize(items.size())
+                        .maxCols(maxCols).build();
+                Queue<IterationArrays> iterationArraysQueue = iterationArrays.getIfPresent(key);
+                IterationArrays iterationArrays1;
+                if(iterationArraysQueue == null) {
+                    iterationArraysQueue = new ConcurrentLinkedQueue<>();
+                    iterationArrays.put(key,iterationArraysQueue);
+                    iterationArrays1 = new IterationArrays(items.size(),maxCols,maxWinWordsCols);
+                } else {
+                    if(iterationArraysQueue.isEmpty()) {
+                        iterationArrays1 = new IterationArrays(items.size(),maxCols,maxWinWordsCols);
+
+                    }else {
+                        try {
+                            iterationArrays1 = iterationArraysQueue.remove();
+                            iterationArrays1.initCodes();
+                        } catch (NoSuchElementException e) {
+                            iterationArrays1 = new IterationArrays(items.size(),maxCols);
+                        }
+
+                    }
+                }
+
+                int[][] inputWindowWordsArr = iterationArrays1.inputWindowWordsArr;
+                int[][] inputWindowWordStatuses = iterationArrays1.inputWindowWordStatuses;
+                int[]  currentWindowIndexesArr = iterationArrays1.currentWindowIndexes;
+                double[] alphasArr = iterationArrays1.alphas;
+                int[][] indicesArr = iterationArrays1.indicesArr;
+                int[][]  codesArr = iterationArrays1.codesArr;
+                long[] randomValues = iterationArrays1.randomValues;
+                int[] numLabelsArr = iterationArrays1.numLabels;
+                currentWindowIndexes = Nd4j.createFromArray(currentWindowIndexesArr);
+
+                for (int cnt = 0; cnt < items.size(); cnt++) {
+                    T currentWord = items.get(cnt).getWord();
+                    currentWindowIndexes.putScalar(0, currentWord.getIndex());
+                    currentWindowIndexesArr[0] = currentWord.getIndex();
+                    int[] windowWords = items.get(cnt).getWindowWords().clone();
+                    boolean[] windowStatuses = items.get(cnt).getWordStatuses().clone();
+
+                    for (int i = 0; i < maxWinWordsCols; i++) {
+                        if (i < windowWords.length) {
+                            inputWindowWordsArr[cnt][i] = windowWords[i];
+                            inputWindowWordStatuses[cnt][i] = windowStatuses[i] ? 1 : 0;
+
+                        } else {
+                            inputWindowWordsArr[cnt][i] = -1;
+                            inputWindowWordStatuses[cnt][i] = -1;
+                        }
+                    }
+
+                    long randomValue = items.get(cnt).getRandomValue();
+                    double alpha = items.get(cnt).getAlpha();
+                    alphasArr[cnt] = alpha;
+                    randomValues[cnt] = randomValue;
+                    numLabelsArr[cnt] = items.get(cnt).getNumLabel();
+                    if (items.get(cnt).getNumLabel() > 0)
+                        hasNumLabels = true;
+
+                    if (useHS) {
+                        for (int p = 0; p < currentWord.getCodeLength(); p++) {
+                            if (currentWord.getPoints().get(p) < 0)
+                                continue;
+
+
+                            codesArr[cnt][p] = currentWord.getCodes().get(p);
+                            indicesArr[cnt][p] =  currentWord.getPoints().get(p);
+                        }
+
+                    }
+
+                    if (negative > 0) {
+                        if (syn1Neg == null) {
+                            ((InMemoryLookupTable<T>) lookupTable).initNegative();
+                            syn1Neg = new DeviceLocalNDArray(((InMemoryLookupTable<T>) lookupTable).getSyn1Neg());
+                        }
+                    }
+
+                }
+
+
+                inputWindowWords = Nd4j.createFromArray(inputWindowWordsArr);
+                inputWordsStatuses = Nd4j.createFromArray(inputWindowWordStatuses);
+                numLabelsArray = Nd4j.createFromArray(numLabelsArr);
+                indices = Nd4j.createFromArray(indicesArr);
+                codes = Nd4j.createFromArray(codesArr);
+                alphas = Nd4j.createFromArray(alphasArr);
+                randoms = Nd4j.createFromArray(randomValues);
+                CbowRound cbow = CbowRound.builder()
+                        .target(currentWindowIndexes)
+                        .context(inputWindowWords)
+                        .lockedWords(inputWordsStatuses)
+                        .ngStarter(currentWindowIndexes)
+                        .syn0(syn0.get())
+                        .syn1(useHS ? syn1.get() : Nd4j.empty(syn0.get().dataType()))
+                        .syn1Neg((negative > 0) ? syn1Neg.get() : Nd4j.empty(syn0.get().dataType()))
+                        .expTable(expTable.get())
+                        .negTable(negative > 0 ? table.get() : Nd4j.empty(syn0.get().dataType()))
+                        .indices(useHS ? indices : Nd4j.empty(DataType.INT32))
+                        .codes(useHS ? codes : Nd4j.empty(DataType.INT8))
+                        .nsRounds((int) negative)
+                        .alpha(alphas)
+                        .nextRandom(randoms)
+                        .inferenceVector(inferenceVector != null ? inferenceVector : Nd4j.empty(syn0.get().dataType()))
+                        .numLabels(hasNumLabels ? numLabelsArray : Nd4j.empty(DataType.INT32))
+                        .trainWords(configuration.isTrainElementsVectors())
+                        .numWorkers(workers)
+                        .iterations(useInference ? configuration.getIterations() * configuration.getEpochs() : 1)
+                        .build();
+
+
+                Nd4j.getExecutioner().exec(cbow);
+
+
+                Nd4j.close(currentWindowIndexes,inputWindowWords,alphas,randoms,codes,numLabelsArray,indices);
+                if(iterationArraysQueue.size() < maxQueueSize)
+                    iterationArraysQueue.add(iterationArrays1);
+
+                batches.get().clear();
+                return 0.0;
+            } else {
+                int cnt = 0;
+                T currentWord = items.get(cnt).getWord();
+                int currentWindowIndex = currentWord.getIndex();
+
+                int[] windowWords = items.get(cnt).getWindowWords().clone();
+                boolean[] windowStatuses = items.get(cnt).getWordStatuses().clone();
+                byte[] codes = new byte[currentWord.getCodeLength()];
+                int[] points = new int[currentWord.getCodeLength()];
+                long randomValue = items.get(cnt).getRandomValue();
+                double alpha = items.get(cnt).getAlpha();
+                int numLabels = items.get(cnt).getNumLabel();
+                int[] inputStatuses = new int[windowWords.length];
+                for (int i = 0; i < windowWords.length; ++i) {
+                    if (i < windowStatuses.length)
+                        inputStatuses[i] = windowStatuses[i] ? 1 : 0;
+                    else
+                        inputStatuses[i] = -1;
+                }
+
+                if (useHS) {
+                    for (int p = 0; p < currentWord.getCodeLength(); p++) {
+                        if (currentWord.getPoints().get(p) < 0)
+                            continue;
+
+                        codes[p] = currentWord.getCodes().get(p);
+                        points[p] = currentWord.getPoints().get(p);
+                    }
+
+                }
+
+                if (negative > 0) {
+                    if (syn1Neg == null) {
+                        ((InMemoryLookupTable<T>) lookupTable).initNegative();
+                        syn1Neg = new DeviceLocalNDArray(((InMemoryLookupTable<T>) lookupTable).getSyn1Neg());
+                    }
+                } else {
+                    syn1Neg.set(Nd4j.empty(syn0.get().dataType()));
+                }
+
+                CbowInference cbowInference = CbowInference.builder()
+                        .target(currentWord.getIndex())
+                        .ngStarter(currentWord.getIndex())
+                        .negTable(table.get() == null ? Nd4j.empty(syn0.get().dataType()) : table.get())
+                        .expTable(expTable.get() == null ? Nd4j.empty(syn0.get().dataType()) : expTable.get())
+                        .syn0(syn0.get())
+                        .syn1(!useHS ? Nd4j.empty(syn0.get().dataType()) : syn1.get())
+                        .syn1Neg(syn1Neg.get())
+                        .alpha(alpha)
+                        .context(windowWords == null ? new int[0] : windowWords)
+                        .indices(useHS || useNegative ? points : new int[0])
+                        .codes(useHS ? codes : new byte[0])
+                        .lockedWords(inputStatuses)
+                        .randomValue((int) randomValue)
+                        .numWorkers(workers)
+                        .numLabels(numLabels)
+                        .nsRounds(useNegative ? (int) negative : 0)
+                        .preciseMode(configuration.isPreciseMode())
+                        .inferenceVector(inferenceVector != null ? inferenceVector : Nd4j.empty(syn0.get().dataType()))
+                        .iterations(useInference ? configuration.getIterations() * configuration.getEpochs() : 1)
+                        .build();
+
+
+                Nd4j.getExecutioner().exec(cbowInference);
+            }
+
+        }
+        return 0.0;
+    }
+
+    public void cbow(int i, List<T> sentence, int b, AtomicLong nextRandom, double alpha, int currentWindow,
+                     List<BatchItem<T>> batch) {
+
+        int end = window * 2 + 1 - b;
+
+        T currentWord = sentence.get(i);
+
+        List<Integer> intsList = new ArrayList<>();
+        List<Boolean> statusesList = new ArrayList<>();
+        for (int a = b; a < end; a++) {
+            if (a != currentWindow) {
+                int c = i - currentWindow + a;
+                if (c >= 0 && c < sentence.size()) {
+                    T lastWord = sentence.get(c);
+
+                    intsList.add(lastWord.getIndex());
+                    statusesList.add(lastWord.isLocked());
+                }
+            }
+        }
+
+        int[] windowWords = new int[intsList.size()];
+        boolean[] statuses = new boolean[intsList.size()];
+        for (int x = 0; x < windowWords.length; x++) {
+            windowWords[x] = intsList.get(x);
+            statuses[x] = statusesList.get(x);
+        }
+
+        BatchItem<T> batchItem = new BatchItem<>(currentWord,windowWords,statuses,nextRandom.get(),alpha);
+        batch.add(batchItem);
+        iterateBatchesIfReady(batch);
+
+
+    }
+
+    private double iterateBatchesIfReady(List<BatchItem<T>> batch) {
+        double score = 0.0;
+        if(batches.get() == null) {
+            batches.set(batch);
+        }
+        else
+            batches.get().addAll(batch);
+
+        if(batches.get().size() >= configuration.getBatchSize()) {
+            score = doExec(batches.get(),null);
+            batches.get().clear();
+
+        }
+        return score;
+    }
+
+    public Sequence<T> applySubsampling(@NonNull Sequence<T> sequence, @NonNull AtomicLong nextRandom) {
+        Sequence<T> result = new Sequence<>();
+
+        // subsampling implementation, if subsampling threshold met, just continue to next element
+        if (sampling > 0) {
+            result.setSequenceId(sequence.getSequenceId());
+            if (sequence.getSequenceLabels() != null)
+                result.setSequenceLabels(sequence.getSequenceLabels());
+            if (sequence.getSequenceLabel() != null)
+                result.setSequenceLabel(sequence.getSequenceLabel());
+
+            for (T element : sequence.getElements()) {
+                double numWords = vocabCache.totalWordOccurrences();
+                double ran = (Math.sqrt(element.getElementFrequency() / (sampling * numWords)) + 1)
+                        * (sampling * numWords) / element.getElementFrequency();
+
+                nextRandom.set(Math.abs(nextRandom.get() * 25214903917L + 11));
+
+                if (ran < (nextRandom.get() & 0xFFFF) / (double) 65536) {
+                    continue;
+                }
+                result.addElement(element);
+            }
+            return result;
+        } else
+            return sequence;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/IterationArrays.java

@@ -0,0 +1,73 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.deeplearning4j.models.embeddings.learning.impl.elements;
+
+import lombok.Data;
+
+@Data
+public class IterationArrays {
+    private int itemSize;
+    private int maxCols;
+    private int maxWinWordsCols;
+    int[][] indicesArr;
+    int[][] codesArr;
+    long[] randomValues;
+    int[] ngStarters;
+    double[] alphas;
+    int[] targets;
+    int[][] inputWindowWordsArr;
+    int[][] inputWindowWordStatuses;
+    int[] currentWindowIndexes;
+    int[] numLabels;
+
+    public IterationArrays(int itemSize, int maxCols,int maxWinWordsCols) {
+        this.maxWinWordsCols = maxWinWordsCols;
+        this.itemSize = itemSize;
+        this.maxCols = maxCols;
+        indicesArr = new int[itemSize][maxCols];
+        codesArr = new int[itemSize][maxCols];
+        currentWindowIndexes = new int[itemSize];
+        inputWindowWordsArr = new int[itemSize][maxWinWordsCols];
+        inputWindowWordStatuses = new int[itemSize][maxWinWordsCols];
+        randomValues = new long[itemSize];
+        ngStarters = new int[itemSize];
+        alphas = new double[itemSize];
+        targets = new int[itemSize];
+        numLabels = new int[itemSize];
+        initCodes();
+    }
+
+    public IterationArrays(int itemSize, int maxCols) {
+        this(itemSize,maxCols,0);
+    }
+
+    /**
+     * USed to initialize codes to the -1 default value.
+     * This method should be called when reusing an iteration arrays object.
+     */
+    public void initCodes() {
+        for (int i = 0; i < codesArr.length; i++) {
+            for (int j = 0; j < codesArr[0].length; j++) {
+                codesArr[i][j] = -1;
+            }
+        }
+    }
+
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/IterationArraysKey.java

@@ -0,0 +1,34 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.deeplearning4j.models.embeddings.learning.impl.elements;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@Builder
+@NoArgsConstructor
+public class IterationArraysKey {
+    private int itemSize;
+    private int maxCols;
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/SkipGram.java

@@ -0,0 +1,515 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings.learning.impl.elements;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.deeplearning4j.config.DL4JSystemProperties;
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable;
+import org.deeplearning4j.models.embeddings.learning.ElementsLearningAlgorithm;
+import org.deeplearning4j.models.embeddings.loader.VectorsConfiguration;
+import org.deeplearning4j.models.sequencevectors.interfaces.SequenceIterator;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.nd4j.linalg.api.buffer.DataType;
+import org.nd4j.linalg.api.memory.MemoryWorkspace;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.api.ops.impl.nlp.SkipGramInference;
+import org.nd4j.linalg.api.ops.impl.nlp.SkipGramRound;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.util.DeviceLocalNDArray;
+import org.nd4j.shade.guava.cache.Cache;
+import org.nd4j.shade.guava.cache.CacheBuilder;
+import org.nd4j.shade.guava.cache.Weigher;
+
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Slf4j
+public class SkipGram<T extends SequenceElement> implements ElementsLearningAlgorithm<T> {
+    protected VocabCache<T> vocabCache;
+    protected WeightLookupTable<T> lookupTable;
+    protected VectorsConfiguration configuration;
+    protected int window;
+    protected boolean useAdaGrad;
+    protected double negative;
+    protected double sampling;
+    protected int[] variableWindows;
+    protected int vectorLength;
+    protected int maxQueueSize = Integer.parseInt(System.getProperty(DL4JSystemProperties.NLP_QUEUE_SIZE,"1000"));
+
+    private Cache<IterationArraysKey, Queue<IterationArrays>> iterationArrays = CacheBuilder.newBuilder()
+            .maximumSize(Integer.parseInt(System.getProperty(DL4JSystemProperties.NLP_CACHE_SIZE,"1000")))
+            .weakKeys()
+            .expireAfterWrite(Duration.ofMinutes(5))
+            .build();
+
+    protected int workers = Runtime.getRuntime().availableProcessors();
+
+
+
+    public int getWorkers() {
+        return workers;
+    }
+
+    public void setWorkers(int workers) {
+        this.workers = workers;
+    }
+
+    @Getter
+    @Setter
+    protected DeviceLocalNDArray syn0, syn1, syn1Neg, table, expTable;
+
+    protected ThreadLocal<List<BatchItem<T>>> batches = new ThreadLocal<>();
+
+
+    /**
+     * Dummy construction is required for reflection
+     */
+    public SkipGram() {
+    }
+
+    public List<BatchItem<T>> getBatch() {
+        if (batches.get() == null)
+            batches.set(new ArrayList<>());
+        return batches.get();
+
+    }
+
+
+    /**
+     * Returns implementation code name
+     *
+     * @return
+     */
+    @Override
+    public String getCodeName() {
+        return "SkipGram";
+    }
+
+    /**
+     * SkipGram initialization over given vocabulary and WeightLookupTable
+     *
+     * @param vocabCache
+     * @param lookupTable
+     * @param configuration
+     */
+    @Override
+    public void configure(@NonNull VocabCache<T> vocabCache, @NonNull WeightLookupTable<T> lookupTable,
+                          @NonNull VectorsConfiguration configuration) {
+        this.vocabCache = vocabCache;
+        this.lookupTable = lookupTable;
+        this.configuration = configuration;
+        if (configuration.getNegative() > 0) {
+            if (((InMemoryLookupTable<T>) lookupTable).getSyn1Neg() == null) {
+                log.info("Initializing syn1Neg...");
+                ((InMemoryLookupTable<T>) lookupTable).setUseHS(configuration.isUseHierarchicSoftmax());
+                ((InMemoryLookupTable<T>) lookupTable).setNegative(configuration.getNegative());
+                lookupTable.resetWeights(false);
+            }
+        }
+
+        this.syn0 = new DeviceLocalNDArray(((InMemoryLookupTable<T>) lookupTable).getSyn0());
+        this.syn1 = new DeviceLocalNDArray(((InMemoryLookupTable<T>) lookupTable).getSyn1());
+        this.syn1Neg = new DeviceLocalNDArray(((InMemoryLookupTable<T>) lookupTable).getSyn1Neg());
+        this.expTable = new DeviceLocalNDArray(Nd4j.create(((InMemoryLookupTable<T>) lookupTable).getExpTable(),
+                new long[]{((InMemoryLookupTable<T>) lookupTable).getExpTable().length}, syn0.get() == null ? DataType.DOUBLE
+                        : syn0.get().dataType()));
+        this.table = new DeviceLocalNDArray(((InMemoryLookupTable<T>) lookupTable).getTable());
+
+
+        this.window = configuration.getWindow();
+        this.useAdaGrad = configuration.isUseAdaGrad();
+        this.negative = configuration.getNegative();
+        this.sampling = configuration.getSampling();
+        this.variableWindows = configuration.getVariableWindows();
+        this.workers = configuration.getWorkers();
+        this.vectorLength = configuration.getLayersSize();
+    }
+
+    /**
+     * SkipGram doesn't involve any pretraining
+     *
+     * @param iterator
+     */
+    @Override
+    public void pretrain(SequenceIterator<T> iterator) {
+        // no-op
+    }
+
+    public Sequence<T> applySubsampling(@NonNull Sequence<T> sequence, @NonNull AtomicLong nextRandom) {
+        Sequence<T> result = new Sequence<>();
+
+        // subsampling implementation, if subsampling threshold met, just continue to next element
+        if (sampling > 0) {
+            result.setSequenceId(sequence.getSequenceId());
+            if (sequence.getSequenceLabels() != null)
+                result.setSequenceLabels(sequence.getSequenceLabels());
+            if (sequence.getSequenceLabel() != null)
+                result.setSequenceLabel(sequence.getSequenceLabel());
+
+            for (T element : sequence.getElements()) {
+                double numWords = vocabCache.totalWordOccurrences();
+                double ran = (Math.sqrt(element.getElementFrequency() / (sampling * numWords)) + 1)
+                        * (sampling * numWords) / element.getElementFrequency();
+
+                nextRandom.set(Math.abs(nextRandom.get() * 25214903917L + 11));
+
+                if (ran < (nextRandom.get() & 0xFFFF) / (double) 65536) {
+                    continue;
+                }
+                result.addElement(element);
+            }
+            return result;
+        } else
+            return sequence;
+    }
+
+
+    /**
+     * Learns sequence using SkipGram algorithm
+     *
+     * @param sequence
+     * @param nextRandom
+     * @param learningRate
+     */
+    @Override
+    public double learnSequence(@NonNull Sequence<T> sequence, @NonNull AtomicLong nextRandom, double learningRate) {
+        Sequence<T> tempSequence = sequence;
+        if (sampling > 0)
+            tempSequence = applySubsampling(sequence, nextRandom);
+
+        double score = 0.0;
+
+        int currentWindow = window;
+
+        if (variableWindows != null && variableWindows.length != 0) {
+            currentWindow = variableWindows[RandomUtils.nextInt(0, variableWindows.length)];
+        }
+        for (int i = 0; i < tempSequence.getElements().size(); i++) {
+            nextRandom.set(Math.abs(nextRandom.get() * 25214903917L + 11));
+            score = skipGram(i, tempSequence.getElements(), (int) nextRandom.get() % currentWindow, nextRandom,
+                    learningRate, currentWindow);
+        }
+
+        if (getBatch() != null && getBatch().size() >= configuration.getBatchSize()) {
+            doExec(getBatch(),null);
+            getBatch().clear();
+        }
+
+        return score;
+    }
+
+
+    public void clearBatch() {
+        getBatch().clear();
+    }
+
+    @Override
+    public void finish() {
+        if (batches != null && batches.get() != null && !batches.get().isEmpty()) {
+            iterateSample(null);
+            clearBatch();
+        }
+    }
+
+    @Override
+    public void finish(INDArray inferenceVector) {
+        if (batches != null && batches.get() != null && !batches.get().isEmpty()) {
+            iterateSample(null);
+            clearBatch();
+        }
+    }
+
+    /**
+     * SkipGram has no reasons for early termination ever.
+     *
+     * @return
+     */
+    @Override
+    public boolean isEarlyTerminationHit() {
+        return false;
+    }
+
+    public void addBatchItem(BatchItem<T> batchItem) {
+        getBatch().add(batchItem);
+    }
+
+    private double skipGram(int i, List<T> sentence, int b, AtomicLong nextRandom, double alpha, int currentWindow) {
+        final T word = sentence.get(i);
+        if (word == null || sentence.isEmpty() || word.isLocked())
+            return 0.0;
+
+        double score = 0.0;
+        int end = currentWindow * 2 + 1 - b;
+        for (int a = b; a < end; a++) {
+            if (a != currentWindow) {
+                int c = i - currentWindow + a;
+                if (c >= 0 && c < sentence.size()) {
+                    T lastWord = sentence.get(c);
+                    nextRandom.set(Math.abs(nextRandom.get() * 25214903917L + 11));
+                    BatchItem<T> batchItem = new BatchItem<>(word, lastWord, nextRandom.get(), alpha);
+                    addBatchItem(batchItem);
+
+                }
+            }
+        }
+
+        return score;
+    }
+
+
+    public double iterateSample(BatchItem<T> item) {
+        double score = 0.0;
+
+        List<BatchItem<T>> items = getBatch();
+        if(item != null) {
+            items.add(item);
+            if(items.size() >= configuration.getBatchSize()) {
+                score = doExec(items, null);
+            }
+        } else if(item == null && !items.isEmpty()) {
+            if(items.size() >= configuration.getBatchSize()) {
+                score = doExec(items, null);
+            }
+        }
+
+        return score;
+
+    }
+
+    public  Double doExec(List<BatchItem<T>> items,INDArray inferenceVector) {
+        try(MemoryWorkspace workspace = Nd4j.getWorkspaceManager().scopeOutOfWorkspaces()) {
+            if (items.size() > 1) {
+                INDArray targetArray = null;
+                INDArray ngStarterArray = null;
+                INDArray alphasArray = null;
+                INDArray randomValuesArr = null;
+                int maxCols = 1;
+                for (int i = 0; i < items.size(); i++) {
+                    int curr = items.get(i).getWord().getCodeLength();
+                    if (curr > maxCols)
+                        maxCols = curr;
+                }
+
+                IterationArraysKey key = IterationArraysKey.builder()
+                        .itemSize(items.size())
+                        .maxCols(maxCols).build();
+                Queue<IterationArrays> iterationArraysQueue = iterationArrays.getIfPresent(key);
+                IterationArrays iterationArrays1;
+                if(iterationArraysQueue == null) {
+                    iterationArraysQueue = new ConcurrentLinkedQueue<>();
+                    iterationArrays.put(key,iterationArraysQueue);
+                    iterationArrays1 = new IterationArrays(items.size(),maxCols);
+                } else {
+                    if(iterationArraysQueue.isEmpty()) {
+                        iterationArrays1 = new IterationArrays(items.size(),maxCols);
+
+                    }else {
+                        try {
+                            iterationArrays1 = iterationArraysQueue.remove();
+                            iterationArrays1.initCodes();
+                        }catch(NoSuchElementException e) {
+                            iterationArrays1 = new IterationArrays(items.size(),maxCols);
+                        }
+                    }
+                }
+
+
+                int[][] indicesArr = iterationArrays1.indicesArr;
+                int[][]  codesArr = iterationArrays1.codesArr;
+                //use -1 as padding for codes that are not actually valid for a given row
+                INDArray codes = null;
+                INDArray indices = null;
+
+                long[] randomValues = iterationArrays1.randomValues;
+                double[] alphas = iterationArrays1.alphas;
+                int[] targets = iterationArrays1.targets;
+                int[] ngStarters = iterationArrays1.ngStarters;
+                for (int cnt = 0; cnt < items.size(); cnt++) {
+
+                    T w1 = items.get(cnt).getWord();
+                    T lastWord = items.get(cnt).getLastWord();
+                    randomValues[cnt] = items.get(cnt).getRandomValue();
+                    double alpha = items.get(cnt).getAlpha();
+
+                    if (w1 == null || lastWord == null || (lastWord.getIndex() < 0 && inferenceVector == null)
+
+                            || w1.getIndex() == lastWord.getIndex() || w1.getLabel().equals("STOP")
+                            || lastWord.getLabel().equals("STOP") || w1.getLabel().equals("UNK")
+                            || lastWord.getLabel().equals("UNK")) {
+                        continue;
+                    }
+
+
+                    int target = lastWord.getIndex();
+                    int ngStarter = w1.getIndex();
+
+
+                    targets[cnt] = target;
+                    ngStarters[cnt] = ngStarter;
+                    alphas[cnt] = alpha;
+
+                    if (configuration.isUseHierarchicSoftmax()) {
+                        for (int i = 0; i < w1.getCodeLength(); i++) {
+                            int code = w1.getCodes().get(i);
+                            int point = w1.getPoints().get(i);
+                            if (point >= vocabCache.numWords() || point < 0)
+                                continue;
+                            codesArr[cnt][i] = code;
+                            indicesArr[cnt][i] = point;
+                        }
+
+                    }
+
+                    //negative sampling
+                    if (negative > 0) {
+                        if (syn1Neg == null) {
+                            ((InMemoryLookupTable<T>) lookupTable).initNegative();
+                            syn1Neg = new DeviceLocalNDArray(((InMemoryLookupTable<T>) lookupTable).getSyn1Neg());
+                        }
+                    }
+                }
+
+                alphasArray = Nd4j.createFromArray(alphas);
+                if(negative > 0)
+                    ngStarterArray = Nd4j.createFromArray(ngStarters);
+                randomValuesArr = Nd4j.createFromArray(randomValues);
+                targetArray = Nd4j.createFromArray(targets);
+                if(configuration.isUseHierarchicSoftmax())
+                    codes = Nd4j.createFromArray(codesArr);
+                if(configuration.isUseHierarchicSoftmax())
+                    indices = Nd4j.createFromArray(indicesArr);
+
+                SkipGramRound sg = SkipGramRound.builder()
+                        .target(targetArray)
+                        .expTable(expTable.get())
+                        .ngStarter((negative > 0) ? ngStarterArray : Nd4j.empty(DataType.INT32))
+                        .syn0(syn0.get())
+                        .syn1(configuration.isUseHierarchicSoftmax() ? syn1.get() : Nd4j.empty(syn0.get().dataType()))
+                        .syn1Neg((negative > 0) ? syn1Neg.get() : Nd4j.empty(syn0.get().dataType()))
+                        .negTable((negative > 0) ? table.get() : Nd4j.empty(syn0.get().dataType()))
+                        .indices(configuration.isUseHierarchicSoftmax() ? indices : Nd4j.empty(DataType.INT32))
+                        .codes(configuration.isUseHierarchicSoftmax() ? codes: Nd4j.empty(DataType.INT8))
+                        .alpha(alphasArray)
+                        .randomValue(randomValuesArr)
+                        .inferenceVector(inferenceVector != null ? inferenceVector : Nd4j.empty(syn0.get().dataType()))
+                        .preciseMode(configuration.isPreciseMode())
+                        .numWorkers(workers)
+                        .iterations(inferenceVector != null ? configuration.getIterations() * configuration.getEpochs() : 1)
+                        .build();
+
+                Nd4j.getExecutioner().exec(sg);
+                items.clear();
+
+                sg.inputArguments().clear();
+                Nd4j.close(targetArray,codes,indices,alphasArray,ngStarterArray,randomValuesArr);
+                if(iterationArraysQueue.size() < maxQueueSize)
+                    iterationArraysQueue.add(iterationArrays1);
+
+            } else {
+                int cnt = 0;
+
+                T w1 = items.get(cnt).getWord();
+                T lastWord = items.get(cnt).getLastWord();
+                byte[] codes = new byte[w1.getCodeLength()];
+                int[] indices = new int[w1.getCodeLength()];
+
+                double alpha = items.get(cnt).getAlpha();
+
+                if (w1 == null || lastWord == null || (lastWord.getIndex() < 0 && inferenceVector == null)
+                        || w1.getIndex() == lastWord.getIndex() || w1.getLabel().equals("STOP")
+                        || lastWord.getLabel().equals("STOP") || w1.getLabel().equals("UNK")
+                        || lastWord.getLabel().equals("UNK")) {
+                    return 0.0;
+                }
+
+                int target = lastWord.getIndex();
+                int ngStarter = w1.getIndex();
+
+
+                if (configuration.isUseHierarchicSoftmax()) {
+
+                    for (int i = 0; i < w1.getCodeLength(); i++) {
+                        int code = w1.getCodes().get(i);
+                        int point = w1.getPoints().get(i);
+                        if (point >= vocabCache.numWords() || point < 0)
+                            continue;
+                        if (i < w1.getCodeLength()) {
+                            codes[i] = (byte) code;
+                            indices[i] = point;
+                        }
+
+                    }
+
+                }
+
+                //negative sampling
+                if (negative > 0) {
+                    if (syn1Neg == null) {
+                        ((InMemoryLookupTable<T>) lookupTable).initNegative();
+                        syn1Neg = new DeviceLocalNDArray(((InMemoryLookupTable<T>) lookupTable).getSyn1Neg());
+                    }
+                }
+
+
+                SkipGramInference sg = SkipGramInference.builder()
+                        .inferenceVector(inferenceVector != null ? inferenceVector : Nd4j.empty(syn0.get().dataType()))
+                        .randomValue((int) items.get(0).getRandomValue())
+                        .syn0(syn0.get())
+                        .negTable((negative > 0) ? table.get() : Nd4j.empty(syn0.get().dataType()))
+                        .expTable(expTable.get())
+                        .syn1(configuration.isUseHierarchicSoftmax() ? syn1.get() : Nd4j.empty(syn0.get().dataType()))
+                        .syn1Neg((negative > 0) ? syn1Neg.get() : Nd4j.empty(syn0.get().dataType()))
+                        .negTable((negative > 0) ? table.get() : Nd4j.empty(syn0.get().dataType()))
+                        .alpha(new double[]{alpha})
+                        .iteration(1)
+
+                        .ngStarter(ngStarter)
+                        .indices(indices)
+                        .target(target)
+                        .codes(codes)
+                        .preciseMode(configuration.getPreciseMode())
+                        .numWorkers(configuration.getWorkers())
+                        .build();
+
+                Nd4j.getExecutioner().exec(sg);
+                items.clear();
+
+            }
+            return 0.0;
+
+        }
+
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/sequence/DBOW.java

@@ -0,0 +1,263 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings.learning.impl.sequence;
+
+import lombok.NonNull;
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.embeddings.learning.ElementsLearningAlgorithm;
+import org.deeplearning4j.models.embeddings.learning.SequenceLearningAlgorithm;
+import org.deeplearning4j.models.embeddings.learning.impl.elements.BatchItem;
+import org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram;
+import org.deeplearning4j.models.embeddings.loader.VectorsConfiguration;
+import org.deeplearning4j.models.sequencevectors.interfaces.SequenceIterator;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.nd4j.linalg.api.buffer.DataBuffer;
+import org.nd4j.linalg.api.memory.MemoryWorkspace;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.api.rng.Random;
+import org.nd4j.linalg.factory.Nd4j;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class DBOW<T extends SequenceElement> implements SequenceLearningAlgorithm<T> {
+    protected VocabCache<T> vocabCache;
+    protected WeightLookupTable<T> lookupTable;
+    protected VectorsConfiguration configuration;
+
+
+    protected int window;
+    protected boolean useAdaGrad;
+    protected double negative;
+
+    protected SkipGram<T> skipGram = new SkipGram<>();
+
+    private static final Logger log = LoggerFactory.getLogger(DBOW.class);
+
+    @Override
+    public ElementsLearningAlgorithm<T> getElementsLearningAlgorithm() {
+        return skipGram;
+    }
+
+    public DBOW() {
+
+    }
+
+    @Override
+    public String getCodeName() {
+        return "PV-DBOW";
+    }
+
+    @Override
+    public void configure(@NonNull VocabCache<T> vocabCache, @NonNull WeightLookupTable<T> lookupTable,
+                          @NonNull VectorsConfiguration configuration) {
+        this.vocabCache = vocabCache;
+        this.lookupTable = lookupTable;
+
+        this.window = configuration.getWindow();
+        this.useAdaGrad = configuration.isUseAdaGrad();
+        this.negative = configuration.getNegative();
+        this.configuration = configuration;
+
+        skipGram.configure(vocabCache, lookupTable, configuration);
+    }
+
+    /**
+     * DBOW doesn't involve any pretraining
+     *
+     * @param iterator
+     */
+    @Override
+    public void pretrain(SequenceIterator<T> iterator) {
+
+    }
+
+
+
+    @Override
+    public double learnSequence(@NonNull Sequence<T> sequence, @NonNull AtomicLong nextRandom, double learningRate) {
+        dbow( sequence,  nextRandom, learningRate);
+        return 0;
+    }
+
+    /**
+     * DBOW has no reasons for early termination
+     * @return
+     */
+    @Override
+    public boolean isEarlyTerminationHit() {
+        return false;
+    }
+
+
+
+
+
+    protected void dbow(Sequence<T> sequence, AtomicLong nextRandom, double alpha) {
+        dbow(sequence,nextRandom,alpha,null);
+
+    }
+
+    protected void dbow(Sequence<T> sequence, AtomicLong nextRandom, double alpha,INDArray inferenceVector) {
+        List<T> sentence = skipGram.applySubsampling(sequence, nextRandom).getElements();
+
+
+        if (sequence.getSequenceLabel() == null)
+            return;
+
+        List<T> labels = new ArrayList<>();
+        labels.addAll(sequence.getSequenceLabels());
+
+        if (sentence.isEmpty() || labels.isEmpty())
+            return;
+
+
+
+        if (sequence.getSequenceLabel() == null)
+            return;
+
+
+
+        if (sentence.isEmpty() || labels.isEmpty())
+            return;
+
+        List<BatchItem<T>> batches = inferenceVector != null ?  new ArrayList<>() : skipGram.getBatch();
+        for (T lastWord : labels) {
+            for (T word : sentence) {
+                if (word == null)
+                    continue;
+
+                nextRandom.set(Math.abs(nextRandom.get() * 25214903917L + 11));
+
+                BatchItem<T> batchItem = new BatchItem<>(word,lastWord,nextRandom.get(),alpha);
+                if(inferenceVector != null)
+                    batches.add(batchItem);
+                else skipGram.addBatchItem(batchItem);
+
+
+            }
+        }
+
+
+        if(inferenceVector != null)
+            skipGram.doExec(batches,inferenceVector);
+
+        if (skipGram != null && skipGram.getBatch() != null && skipGram.getBatch() != null
+                && skipGram.getBatch().size() >= configuration.getBatchSize()) {
+            skipGram.doExec(skipGram.getBatch(),null);
+            skipGram.clearBatch();
+        }
+    }
+
+    /**
+     * This method does training on previously unseen paragraph, and returns inferred vector
+     *
+     * @param sequence
+     * @param nextRandom
+     * @param learningRate
+     * @return
+     */
+    @Override
+    public INDArray inferSequence(INDArray inferenceVector, Sequence<T> sequence, long nextRandom, double learningRate, double minLearningRate,
+                                  int iterations) {
+        AtomicLong nr = new AtomicLong(nextRandom);
+        if (sequence.isEmpty())
+            return null;
+
+
+        INDArray ret = inferenceVector;
+        dbow(sequence, nr, learningRate, ret);
+
+
+        return ret;
+    }
+
+
+    /**
+     * This method does training on previously unseen paragraph, and returns inferred vector
+     *
+     * @param sequence
+     * @param nextRandom
+     * @param learningRate
+     * @return
+     */
+    @Override
+    public INDArray inferSequence(Sequence<T> sequence, long nextRandom, double learningRate, double minLearningRate,
+                                  int iterations) {
+        if (sequence.isEmpty())
+            return null;
+
+
+
+        //when workers are > 1 the openmp in the scalar op can cause a crash
+        //set to 1 to workaround
+        int numThreadsOriginal = Nd4j.getEnvironment().maxThreads();
+        if(configuration.getWorkers() > 1) {
+            Nd4j.getEnvironment().setMaxThreads(1);
+        }
+
+
+        try(MemoryWorkspace workspace = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) {
+            Random random = Nd4j.getRandomFactory().getNewRandomInstance(configuration.getSeed() * sequence.hashCode(),
+                    lookupTable.layerSize() + 1);
+
+
+
+            INDArray ret = Nd4j.createUninitializedDetached(this.lookupTable.getWeights().dataType(),lookupTable.layerSize());
+            Nd4j.rand(ret,random);
+            DataBuffer subiDetached = Nd4j.createBufferDetached(new double[] {0.5});
+            DataBuffer diviDetached = Nd4j.createBufferDetached(new int[] {lookupTable.layerSize()});
+            INDArray subi = Nd4j.create(subiDetached,1);
+            INDArray divi = Nd4j.create(diviDetached,1);
+            ret.subi(subi).divi(divi);
+            if(configuration.getWorkers() > 1) {
+                Nd4j.getEnvironment().setMaxThreads(numThreadsOriginal);
+            }
+
+            //close since we don't have a deallocator for random instances
+            random.close();
+
+            Nd4j.close(subi,divi);
+
+            return inferSequence(ret,sequence,nextRandom,learningRate,minLearningRate,iterations);
+        }
+
+    }
+
+    @Override
+    public void finish() {
+        if (skipGram != null && skipGram.getBatch() != null && !skipGram.getBatch().isEmpty()) {
+            skipGram.finish();
+        }
+    }
+
+    @Override
+    public void finish(INDArray inferenceVector) {
+        if (skipGram != null && skipGram.getBatch() != null && !skipGram.getBatch().isEmpty()) {
+            skipGram.finish(inferenceVector);
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/sequence/DM.java

@@ -0,0 +1,273 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings.learning.impl.sequence;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable;
+import org.deeplearning4j.models.embeddings.learning.ElementsLearningAlgorithm;
+import org.deeplearning4j.models.embeddings.learning.SequenceLearningAlgorithm;
+import org.deeplearning4j.models.embeddings.learning.impl.elements.BatchItem;
+import org.deeplearning4j.models.embeddings.learning.impl.elements.CBOW;
+import org.deeplearning4j.models.embeddings.loader.VectorsConfiguration;
+import org.deeplearning4j.models.sequencevectors.interfaces.SequenceIterator;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.nd4j.linalg.api.memory.MemoryWorkspace;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.api.rng.Random;
+import org.nd4j.linalg.factory.Nd4j;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Slf4j
+public class DM<T extends SequenceElement> implements SequenceLearningAlgorithm<T> {
+    private VocabCache<T> vocabCache;
+    private WeightLookupTable<T> lookupTable;
+    private VectorsConfiguration configuration;
+
+
+    protected int window;
+    protected boolean useAdaGrad;
+    protected double negative;
+    protected double sampling;
+
+    protected double[] expTable;
+
+    protected INDArray syn0, syn1, syn1Neg, table;
+
+    private CBOW<T> cbow = new CBOW<>();
+
+    @Override
+    public ElementsLearningAlgorithm<T> getElementsLearningAlgorithm() {
+        return cbow;
+    }
+
+    @Override
+    public String getCodeName() {
+        return "PV-DM";
+    }
+
+    @Override
+    public void configure(@NonNull VocabCache<T> vocabCache, @NonNull WeightLookupTable<T> lookupTable,
+                          @NonNull VectorsConfiguration configuration) {
+        this.vocabCache = vocabCache;
+        this.lookupTable = lookupTable;
+        this.configuration = configuration;
+
+        cbow.configure(vocabCache, lookupTable, configuration);
+
+        this.window = configuration.getWindow();
+        this.useAdaGrad = configuration.isUseAdaGrad();
+        this.negative = configuration.getNegative();
+        this.sampling = configuration.getSampling();
+
+        this.syn0 = ((InMemoryLookupTable<T>) lookupTable).getSyn0();
+        this.syn1 = ((InMemoryLookupTable<T>) lookupTable).getSyn1();
+        this.syn1Neg = ((InMemoryLookupTable<T>) lookupTable).getSyn1Neg();
+        this.expTable = ((InMemoryLookupTable<T>) lookupTable).getExpTable();
+        this.table = ((InMemoryLookupTable<T>) lookupTable).getTable();
+    }
+
+    @Override
+    public void pretrain(SequenceIterator<T> iterator) {
+        // no-op
+    }
+
+    @Override
+    public double learnSequence(Sequence<T> sequence, AtomicLong nextRandom, double learningRate) {
+        Sequence<T> seq = cbow.applySubsampling(sequence, nextRandom);
+
+        if (sequence.getSequenceLabel() == null)
+            return 0;
+
+        List<T> labels = new ArrayList<>();
+        labels.addAll(sequence.getSequenceLabels());
+
+        if (seq.isEmpty() || labels.isEmpty())
+            return 0;
+
+
+        for (int i = 0; i < seq.size(); i++) {
+            nextRandom.set(Math.abs(nextRandom.get() * 25214903917L + 11));
+            dm(i, seq, (int) nextRandom.get() % window, nextRandom, learningRate, labels,null);
+        }
+
+        return 0;
+    }
+
+
+
+
+
+
+    public void dm(int i, Sequence<T> sequence, int b, AtomicLong nextRandom, double alpha, List<T> labels,INDArray inferenceVector) {
+        int end = window * 2 + 1 - b;
+
+        T currentWord = sequence.getElementByIndex(i);
+
+        List<Integer> intsList = new ArrayList<>();
+        List<Boolean> statusesList = new ArrayList<>();
+        int[] windowWords = new int[intsList.size()];
+        boolean[] statuses = new boolean[intsList.size()];
+        for (int x = 0; x < windowWords.length; x++) {
+            windowWords[x] = intsList.get(x);
+            statuses[x] = false;
+        }
+
+
+        // appending labels indexes
+        if (labels != null)
+            for (T label : labels) {
+                intsList.add(label.getIndex());
+            }
+
+
+        List<BatchItem<T>> batches = inferenceVector != null ? new ArrayList<>() : cbow.getBatch();
+        BatchItem<T> batch = new BatchItem<>(currentWord,windowWords,statuses,nextRandom.get(),alpha);
+
+        for (int a = b; a < end; a++) {
+            if (a != window) {
+                int c = i - window + a;
+                if (c >= 0 && c < sequence.size()) {
+                    T lastWord = sequence.getElementByIndex(c);
+
+                    intsList.add(lastWord.getIndex());
+                    statusesList.add(lastWord.isLocked());
+                    if(inferenceVector != null)
+                        batches.add(batch);
+                    else
+                        cbow.addBatchItem(batch);
+
+                }
+            }
+        }
+
+
+        if(inferenceVector != null) {
+            cbow.doExec(batches,inferenceVector);
+        }
+
+
+        if(inferenceVector == null) {
+            if(cbow != null && cbow.getBatch() != null && cbow.getBatch().size() >= configuration.getBatchSize())
+                finish();
+        }
+
+
+    }
+
+    @Override
+    public boolean isEarlyTerminationHit() {
+        return false;
+    }
+
+    @Override
+    public INDArray inferSequence(INDArray inferenceVector, Sequence<T> sequence, long nextRandom, double learningRate, double minLearningRate, int iterations) {
+        AtomicLong nextRandom2 = new AtomicLong(nextRandom);
+        // we probably don't want subsampling here
+
+        if (sequence.isEmpty())
+            return null;
+
+
+        try(MemoryWorkspace memoryWorkspace = Nd4j.getWorkspaceManager().scopeOutOfWorkspaces()) {
+            Random random = Nd4j.getRandomFactory().getNewRandomInstance(configuration.getSeed() * sequence.hashCode(),
+                    lookupTable.layerSize() + 1);
+
+
+            int numThreadsOriginal = Nd4j.getEnvironment().maxThreads();
+            //when workers are > 1 the openmp in the scalar op can cause a crash
+            //set to 1 to workaround
+            if(configuration.getWorkers() > 1) {
+                Nd4j.getEnvironment().setMaxThreads(1);
+            }
+
+            INDArray ret = Nd4j.createUninitializedDetached(this.lookupTable.getWeights().dataType(),lookupTable.layerSize());
+            Nd4j.rand(ret,random);
+            ret.subi(0.5).divi(lookupTable.layerSize());
+
+            log.info("Inf before: {}", ret);
+            dm(0, sequence, (int) nextRandom2.get() % window, nextRandom2, learningRate,Collections.emptyList(), ret);
+
+            if(configuration.getWorkers() > 1) {
+                Nd4j.getEnvironment().setMaxThreads(numThreadsOriginal);
+            }
+
+            //close since we don't have a deallocator for random instances
+            random.close();
+
+            return ret;
+
+        }
+
+    }
+
+    /**
+     * This method does training on previously unseen paragraph, and returns inferred vector
+     *
+     * @param sequence
+     * @param nr
+     * @param learningRate
+     * @return
+     */
+    @Override
+    public INDArray inferSequence(Sequence<T> sequence, long nr, double learningRate, double minLearningRate,
+                                  int iterations) {
+        AtomicLong nextRandom = new AtomicLong(nr);
+        // we probably don't want subsampling here
+
+        if (sequence.isEmpty())
+            return null;
+
+        Random random = Nd4j.getRandomFactory().getNewRandomInstance(configuration.getSeed() * sequence.hashCode(),
+                lookupTable.layerSize() + 1);
+        INDArray ret = Nd4j.rand(random,lookupTable.getWeights().dataType(),
+                        1, lookupTable.layerSize()).subi(0.5)
+                .divi(lookupTable.layerSize());
+
+        log.info("Inf before: {}", ret);
+        dm(0, sequence, (int) nextRandom.get() % window, nextRandom, learningRate,Collections.emptyList(), ret);
+        random.close();
+
+        return ret;
+    }
+
+
+    @Override
+    public void finish() {
+        if (cbow != null && cbow.getBatch() != null && !cbow.getBatch().isEmpty()) {
+            cbow.finish();
+        }
+    }
+
+    @Override
+    public void finish(INDArray inferenceVector) {
+        if (cbow != null && cbow.getBatch() != null && !cbow.getBatch().isEmpty()) {
+            cbow.finish(inferenceVector);
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/VectorsConfiguration.java

@@ -0,0 +1,732 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings.loader;
+
+import org.apache.commons.codec.binary.Base64;
+import org.nd4j.shade.jackson.annotation.*;
+import org.nd4j.shade.jackson.databind.DeserializationFeature;
+import org.nd4j.shade.jackson.databind.MapperFeature;
+import org.nd4j.shade.jackson.databind.ObjectMapper;
+import org.nd4j.shade.jackson.databind.SerializationFeature;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Objects;
+
+public class VectorsConfiguration implements Serializable {
+
+    // word2vec params
+    private Integer minWordFrequency = 5;
+    private Double learningRate = 0.025;
+    private Double minLearningRate = 0.0001;
+    private Integer layersSize = 200;
+    private Boolean useAdaGrad = false;
+    private Integer batchSize = 512;
+    private Integer iterations = 1;
+    private Integer epochs = 1;
+    private Integer window = 5;
+    private Long seed = 1234L;
+    private Double negative = 0.0d;
+    private Boolean useHierarchicSoftmax = true;
+    private Double sampling = 0.0d;
+    private Integer learningRateDecayWords = 3;
+    private int[] variableWindows;
+
+    private Boolean hugeModelExpected = false;
+    private Boolean useUnknown = false;
+
+    private Integer scavengerActivationThreshold = 2000000;
+    private Integer scavengerRetentionDelay = 3;
+
+    private String elementsLearningAlgorithm;
+    private String sequenceLearningAlgorithm;
+    private String modelUtils;
+
+    private int workers;
+
+    private String tokenizerFactory;
+    private String tokenPreProcessor;
+
+    // this is one-off configuration value specially for NGramTokenizerFactory
+    private Integer nGram;
+
+    private String UNK = "UNK";
+    private String STOP = "STOP";
+
+    private Collection<String> stopList = new ArrayList<>();
+
+    // overall model info
+    private Integer vocabSize;
+
+    // paravec-specific option
+    private Boolean trainElementsVectors;
+    private Boolean trainSequenceVectors;
+    private Boolean allowParallelTokenization;
+    private Boolean preciseWeightInit;
+    private Boolean preciseMode;
+
+    private Integer vectorCalcThreads;
+
+    private static ObjectMapper mapper;
+    private static final Object lock = new Object();
+
+    public VectorsConfiguration() {
+        this.vectorCalcThreads = 1;
+        this.minWordFrequency = 5;
+        this.learningRate = 0.025;
+        this.minLearningRate = 0.0001;
+        this.layersSize = 200;
+        this.useAdaGrad = false;
+        this.batchSize = 512;
+        this.iterations = 1;
+        this.epochs = 1;
+        this.window = 5;
+        this.negative = 0.0d;
+        this.useHierarchicSoftmax = true;
+        this.sampling = 0.0d;
+        this.hugeModelExpected = false;
+        this.useUnknown = false;
+        this.scavengerActivationThreshold = 2000000;
+        this.scavengerRetentionDelay = 3;
+        this.UNK = "UNK";
+        this.STOP = "STOP";
+        this.stopList = new ArrayList<>();
+        this.trainElementsVectors = true;
+        this.trainSequenceVectors = true;
+        this.allowParallelTokenization = false;
+        this.preciseWeightInit = false;
+        this.preciseMode = false;
+        this.workers = Runtime.getRuntime().availableProcessors();
+
+    }
+
+    public Integer getVectorCalcThreads() {
+        return vectorCalcThreads;
+    }
+
+    public void setVectorCalcThreads(Integer vectorCalcThreads) {
+        this.vectorCalcThreads = vectorCalcThreads;
+    }
+
+    public Boolean getUseHierarchicSoftmax() {
+        return useHierarchicSoftmax;
+    }
+
+    public Boolean getHugeModelExpected() {
+        return hugeModelExpected;
+    }
+
+    public Boolean getUseUnknown() {
+        return useUnknown;
+    }
+
+    public int getWorkers() {
+        return workers;
+    }
+
+    public void setWorkers(int workers) {
+        this.workers = workers;
+    }
+
+    public Boolean getTrainElementsVectors() {
+        return trainElementsVectors;
+    }
+
+    public Boolean getTrainSequenceVectors() {
+        return trainSequenceVectors;
+    }
+
+    public Boolean getAllowParallelTokenization() {
+        return allowParallelTokenization;
+    }
+
+    public Boolean getPreciseWeightInit() {
+        return preciseWeightInit;
+    }
+
+    public Boolean getPreciseMode() {
+        return preciseMode;
+    }
+
+    @JsonCreator
+    public VectorsConfiguration(@JsonProperty("minWordFrequency")  Integer minWordFrequency,
+                                @JsonProperty("learningRate") Double learningRate,
+                                @JsonProperty("minLearningRate") Double minLearningRate,
+                                @JsonProperty("layersSize") Integer layersSize,
+                                @JsonProperty("useAdaGrad") Boolean useAdaGrad,
+                                @JsonProperty("batchSize") Integer batchSize,
+                                @JsonProperty("iterations") Integer iterations,
+                                @JsonProperty("epochs") Integer epochs,
+                                @JsonProperty("window") Integer window,
+                                @JsonProperty("seed") Long seed,
+                                @JsonProperty("negative") Double negative,
+                                @JsonProperty("useHierarchicSoftmax") Boolean useHierarchicSoftmax,
+                                @JsonProperty("sampling") Double sampling,
+                                @JsonProperty("learningRateDecayWords") Integer learningRateDecayWords,
+                                @JsonProperty("variableWindows") int[] variableWindows,
+                                @JsonProperty("hugeModelExpected") Boolean hugeModelExpected,
+                                @JsonProperty("useUnknown")  Boolean useUnknown,
+                                @JsonProperty("scavengerActivationThreshold") Integer scavengerActivationThreshold,
+                                @JsonProperty("scavengerRetentionDelay") Integer scavengerRetentionDelay,
+                                @JsonProperty("elementsLearningAlgorithm") String elementsLearningAlgorithm,
+                                @JsonProperty("sequenceLearningAlgorithm") String sequenceLearningAlgorithm,
+                                @JsonProperty("modelUtils") String modelUtils,
+                                @JsonProperty("tokenizerFactory") String tokenizerFactory,
+                                @JsonProperty("tokenPreProcessor") String tokenPreProcessor,
+                                @JsonProperty("nGram") Integer nGram,
+                                @JsonProperty("UNK") String UNK,
+                                @JsonProperty("STOP") String STOP,
+                                @JsonProperty("stopList") Collection<String> stopList,
+                                @JsonProperty("vocabSize")   Integer vocabSize,
+                                @JsonProperty("trainElementsVectors") Boolean trainElementsVectors,
+                                @JsonProperty("trainSequenceVectors") Boolean trainSequenceVectors,
+                                @JsonProperty("allowParallelTokenization") Boolean allowParallelTokenization,
+                                @JsonProperty("preciseWeightInit") Boolean preciseWeightInit,
+                                @JsonProperty("preciseMode") Boolean preciseMode,
+                                @JsonProperty("workers") Integer workers,
+                                @JsonProperty("vectorCalcThreads") Integer vectorCalcThreads) {
+        if(minWordFrequency != null)
+            this.minWordFrequency = minWordFrequency;
+        else
+            this.minWordFrequency = 5;
+        if(learningRate != null)
+            this.learningRate = learningRate;
+        else
+            this.learningRate = 0.025;
+        if(minLearningRate != null)
+            this.minLearningRate = minLearningRate;
+        else
+            this.minLearningRate = 0.0001;
+        if(layersSize != null)
+            this.layersSize = layersSize;
+        else
+            this.layersSize = 200;
+        if(useAdaGrad != null)
+            this.useAdaGrad = useAdaGrad;
+        else
+            this.useAdaGrad = false;
+        if(batchSize != null)
+            this.batchSize = batchSize;
+        else
+            this.batchSize = 512;
+        if(iterations != null)
+            this.iterations = iterations;
+        else
+            this.iterations = 1;
+        if(epochs != null)
+            this.epochs = epochs;
+        else
+            this.epochs = 1;
+        if(window != null)
+            this.window = window;
+        else
+            this.window = 5;
+
+        if(seed != null)
+            this.seed = seed;
+        else
+            this.seed = 0L;
+        if(negative != null)
+            this.negative = negative;
+        else
+            this.negative = 0.0d;
+        if(useHierarchicSoftmax != null)
+            this.useHierarchicSoftmax = useHierarchicSoftmax;
+        else
+            this.useHierarchicSoftmax = true;
+        if(this.sampling != null)
+            this.sampling = sampling;
+        else
+            this.sampling = 0.0d;
+        if(learningRateDecayWords != null)
+            this.learningRateDecayWords = learningRateDecayWords;
+        else
+            this.learningRateDecayWords = 0;
+        this.variableWindows = variableWindows;
+        if(hugeModelExpected != null)
+            this.hugeModelExpected = hugeModelExpected;
+        else
+            this.hugeModelExpected = false;
+        if(this.useUnknown != null)
+            this.useUnknown = useUnknown;
+        else
+            this.useUnknown = false;
+        if(scavengerActivationThreshold != null)
+            this.scavengerActivationThreshold = scavengerActivationThreshold;
+        else
+            this.scavengerActivationThreshold = 2000000;
+        if(scavengerRetentionDelay != null)
+            this.scavengerRetentionDelay = scavengerRetentionDelay;
+        else {
+            this.scavengerRetentionDelay = 3;
+        }
+
+        if(vectorCalcThreads != null) {
+            this.vectorCalcThreads = vectorCalcThreads;
+        } else
+            this.vectorCalcThreads = 1;
+
+        this.elementsLearningAlgorithm = elementsLearningAlgorithm;
+        this.sequenceLearningAlgorithm = sequenceLearningAlgorithm;
+
+        this.modelUtils = modelUtils;
+        this.tokenizerFactory = tokenizerFactory;
+        this.tokenPreProcessor = tokenPreProcessor;
+        this.nGram = nGram;
+
+        if(workers != null)
+            this.workers = workers;
+        else this.workers = Runtime.getRuntime().availableProcessors();
+
+        if(UNK != null)
+            this.UNK = UNK;
+        else
+            this.UNK = "UNK";
+        if(STOP != null)
+            this.STOP = STOP;
+        else
+            this.STOP = "STOP";
+        if(stopList != null)
+            this.stopList = stopList;
+        else
+            this.stopList = new ArrayList<>();
+        this.vocabSize = vocabSize;
+        this.trainElementsVectors = trainElementsVectors;
+        this.trainSequenceVectors = trainSequenceVectors;
+        if(allowParallelTokenization != null)
+            this.allowParallelTokenization = allowParallelTokenization;
+        else
+            this.allowParallelTokenization = false;
+        if(preciseWeightInit != null)
+            this.preciseWeightInit = preciseWeightInit;
+        else
+            this.preciseWeightInit = false;
+        if(preciseMode != null)
+            this.preciseMode = preciseMode;
+        else
+            this.preciseMode = false;
+    }
+
+    private static ObjectMapper mapper() {
+        if (mapper == null) {
+            synchronized (lock) {
+                if (mapper == null) {
+                    mapper = new ObjectMapper();
+                    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+                    mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
+                    mapper.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, false);         //Use order in which fields are defined in classes
+                    mapper.enable(SerializationFeature.INDENT_OUTPUT);
+                    mapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
+                    mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
+                    mapper.setVisibility(PropertyAccessor.CREATOR, JsonAutoDetect.Visibility.ANY);
+                    mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+
+
+                    mapper.configure(SerializationFeature.FAIL_ON_UNWRAPPED_TYPE_IDENTIFIERS, false);
+                    return mapper;
+                }
+            }
+        }
+        return mapper;
+    }
+
+    public String toJson() {
+        ObjectMapper mapper = mapper();
+        try {
+            /*
+                we need JSON as single line to save it at first line of the CSV model file
+                That's ugly method, but its way more memory-friendly then loading whole 10GB json file just to create another 10GB memory array.
+            */
+            return mapper.writeValueAsString(this);
+        } catch (org.nd4j.shade.jackson.core.JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public String toEncodedJson() {
+        Base64 base64 = new Base64(Integer.MAX_VALUE);
+        try {
+            return base64.encodeAsString(this.toJson().getBytes("UTF-8"));
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
+
+    public static VectorsConfiguration fromEncodedJson(String json) {
+        Base64 base64 = new Base64(Integer.MAX_VALUE);
+        try {
+            String decoded = new String(base64.decode(json.getBytes("UTF-8")));
+            return VectorsConfiguration.fromJson(decoded);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static VectorsConfiguration fromJson(String json) {
+        ObjectMapper mapper = mapper();
+        try {
+            VectorsConfiguration ret = mapper.readValue(json, VectorsConfiguration.class);
+            return ret;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public Integer getMinWordFrequency() {
+        return minWordFrequency;
+    }
+
+    public void setMinWordFrequency(Integer minWordFrequency) {
+        this.minWordFrequency = minWordFrequency;
+    }
+
+    public Double getLearningRate() {
+        return learningRate;
+    }
+
+    public void setLearningRate(Double learningRate) {
+        this.learningRate = learningRate;
+    }
+
+    public Double getMinLearningRate() {
+        return minLearningRate;
+    }
+
+    public void setMinLearningRate(Double minLearningRate) {
+        this.minLearningRate = minLearningRate;
+    }
+
+    public Integer getLayersSize() {
+        return layersSize;
+    }
+
+    public void setLayersSize(Integer layersSize) {
+        this.layersSize = layersSize;
+    }
+
+    public Boolean isUseAdaGrad() {
+        return useAdaGrad;
+    }
+
+    public void setUseAdaGrad(Boolean useAdaGrad) {
+        this.useAdaGrad = useAdaGrad;
+    }
+
+    public Integer getBatchSize() {
+        return batchSize;
+    }
+
+    public void setBatchSize(Integer batchSize) {
+        this.batchSize = batchSize;
+    }
+
+    public Integer getIterations() {
+        return iterations;
+    }
+
+    public void setIterations(Integer iterations) {
+        this.iterations = iterations;
+    }
+
+    public Integer getEpochs() {
+        return epochs;
+    }
+
+    public void setEpochs(Integer epochs) {
+        this.epochs = epochs;
+    }
+
+    public Integer getWindow() {
+        return window;
+    }
+
+    public void setWindow(Integer window) {
+        this.window = window;
+    }
+
+    public Long getSeed() {
+        return seed;
+    }
+
+    public void setSeed(Long seed) {
+        this.seed = seed;
+    }
+
+    public Double getNegative() {
+        return negative;
+    }
+
+    public void setNegative(Double negative) {
+        this.negative = negative;
+    }
+
+    public Boolean isUseHierarchicSoftmax() {
+        return useHierarchicSoftmax;
+    }
+
+    public void setUseHierarchicSoftmax(Boolean useHierarchicSoftmax) {
+        this.useHierarchicSoftmax = useHierarchicSoftmax;
+    }
+
+    public Double getSampling() {
+        return sampling;
+    }
+
+    public void setSampling(Double sampling) {
+        this.sampling = sampling;
+    }
+
+    public Integer getLearningRateDecayWords() {
+        return learningRateDecayWords;
+    }
+
+    public void setLearningRateDecayWords(Integer learningRateDecayWords) {
+        this.learningRateDecayWords = learningRateDecayWords;
+    }
+
+    public int[] getVariableWindows() {
+        return variableWindows;
+    }
+
+    public void setVariableWindows(int[] variableWindows) {
+        this.variableWindows = variableWindows;
+    }
+
+    public Boolean isHugeModelExpected() {
+        return hugeModelExpected;
+    }
+
+    public void setHugeModelExpected(Boolean hugeModelExpected) {
+        this.hugeModelExpected = hugeModelExpected;
+    }
+
+    public Boolean isUseUnknown() {
+        return useUnknown;
+    }
+
+    public void setUseUnknown(Boolean useUnknown) {
+        this.useUnknown = useUnknown;
+    }
+
+    public Integer getScavengerActivationThreshold() {
+        return scavengerActivationThreshold;
+    }
+
+    public void setScavengerActivationThreshold(Integer scavengerActivationThreshold) {
+        this.scavengerActivationThreshold = scavengerActivationThreshold;
+    }
+
+    public Integer getScavengerRetentionDelay() {
+        return scavengerRetentionDelay;
+    }
+
+    public void setScavengerRetentionDelay(Integer scavengerRetentionDelay) {
+        this.scavengerRetentionDelay = scavengerRetentionDelay;
+    }
+
+    public String getElementsLearningAlgorithm() {
+        return elementsLearningAlgorithm;
+    }
+
+    public void setElementsLearningAlgorithm(String elementsLearningAlgorithm) {
+        this.elementsLearningAlgorithm = elementsLearningAlgorithm;
+    }
+
+    public String getSequenceLearningAlgorithm() {
+        return sequenceLearningAlgorithm;
+    }
+
+    public void setSequenceLearningAlgorithm(String sequenceLearningAlgorithm) {
+        this.sequenceLearningAlgorithm = sequenceLearningAlgorithm;
+    }
+
+    public String getModelUtils() {
+        return modelUtils;
+    }
+
+    public void setModelUtils(String modelUtils) {
+        this.modelUtils = modelUtils;
+    }
+
+    public String getTokenizerFactory() {
+        return tokenizerFactory;
+    }
+
+    public void setTokenizerFactory(String tokenizerFactory) {
+        this.tokenizerFactory = tokenizerFactory;
+    }
+
+    public String getTokenPreProcessor() {
+        return tokenPreProcessor;
+    }
+
+    public void setTokenPreProcessor(String tokenPreProcessor) {
+        this.tokenPreProcessor = tokenPreProcessor;
+    }
+
+    public Integer getnGram() {
+        return nGram;
+    }
+
+    public void setnGram(Integer nGram) {
+        this.nGram = nGram;
+    }
+
+    public String getUNK() {
+        return UNK;
+    }
+
+    public void setUNK(String UNK) {
+        this.UNK = UNK;
+    }
+
+    public String getSTOP() {
+        return STOP;
+    }
+
+    public void setSTOP(String STOP) {
+        this.STOP = STOP;
+    }
+
+    public Collection<String> getStopList() {
+        return stopList;
+    }
+
+    public void setStopList(Collection<String> stopList) {
+        this.stopList = stopList;
+    }
+
+    public Integer getVocabSize() {
+        return vocabSize;
+    }
+
+    public void setVocabSize(Integer vocabSize) {
+        this.vocabSize = vocabSize;
+    }
+
+    public Boolean isTrainElementsVectors() {
+        return trainElementsVectors;
+    }
+
+    public void setTrainElementsVectors(Boolean trainElementsVectors) {
+        this.trainElementsVectors = trainElementsVectors;
+    }
+
+    public Boolean isTrainSequenceVectors() {
+        return trainSequenceVectors;
+    }
+
+    public void setTrainSequenceVectors(Boolean trainSequenceVectors) {
+        this.trainSequenceVectors = trainSequenceVectors;
+    }
+
+    public Boolean isAllowParallelTokenization() {
+        return allowParallelTokenization;
+    }
+
+    public void setAllowParallelTokenization(Boolean allowParallelTokenization) {
+        this.allowParallelTokenization = allowParallelTokenization;
+    }
+
+    public Boolean isPreciseWeightInit() {
+        return preciseWeightInit;
+    }
+
+    public void setPreciseWeightInit(Boolean preciseWeightInit) {
+        this.preciseWeightInit = preciseWeightInit;
+    }
+
+    public Boolean isPreciseMode() {
+        return preciseMode;
+    }
+
+    public void setPreciseMode(Boolean preciseMode) {
+        this.preciseMode = preciseMode;
+    }
+
+    public static ObjectMapper getMapper() {
+        return mapper;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof VectorsConfiguration)) return false;
+        VectorsConfiguration that = (VectorsConfiguration) o;
+        return Objects.equals(getMinWordFrequency(), that.getMinWordFrequency()) && Objects.equals(getLearningRate(), that.getLearningRate()) && Objects.equals(getMinLearningRate(), that.getMinLearningRate()) && Objects.equals(getLayersSize(), that.getLayersSize()) && Objects.equals(useAdaGrad, that.useAdaGrad) && Objects.equals(getBatchSize(), that.getBatchSize()) && Objects.equals(getIterations(), that.getIterations()) && Objects.equals(getEpochs(), that.getEpochs()) && Objects.equals(getWindow(), that.getWindow()) && Objects.equals(getSeed(), that.getSeed()) && Objects.equals(getNegative(), that.getNegative()) && Objects.equals(useHierarchicSoftmax, that.useHierarchicSoftmax) && Objects.equals(getSampling(), that.getSampling()) && Objects.equals(getLearningRateDecayWords(), that.getLearningRateDecayWords()) && Arrays.equals(getVariableWindows(), that.getVariableWindows()) && Objects.equals(hugeModelExpected, that.hugeModelExpected) && Objects.equals(useUnknown, that.useUnknown) && Objects.equals(getScavengerActivationThreshold(), that.getScavengerActivationThreshold()) && Objects.equals(getScavengerRetentionDelay(), that.getScavengerRetentionDelay()) && Objects.equals(getElementsLearningAlgorithm(), that.getElementsLearningAlgorithm()) && Objects.equals(getSequenceLearningAlgorithm(), that.getSequenceLearningAlgorithm()) && Objects.equals(getModelUtils(), that.getModelUtils()) && Objects.equals(getTokenizerFactory(), that.getTokenizerFactory()) && Objects.equals(getTokenPreProcessor(), that.getTokenPreProcessor()) && Objects.equals(getnGram(), that.getnGram()) && Objects.equals(getUNK(), that.getUNK()) && Objects.equals(getSTOP(), that.getSTOP()) && Objects.equals(getStopList(), that.getStopList()) && Objects.equals(getVocabSize(), that.getVocabSize()) && Objects.equals(trainElementsVectors, that.trainElementsVectors) && Objects.equals(trainSequenceVectors, that.trainSequenceVectors) && Objects.equals(allowParallelTokenization, that.allowParallelTokenization) && Objects.equals(preciseWeightInit, that.preciseWeightInit) && Objects.equals(preciseMode, that.preciseMode);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Objects.hash(getMinWordFrequency(), getLearningRate(), getMinLearningRate(), getLayersSize(), useAdaGrad, getBatchSize(), getIterations(), getEpochs(), getWindow(), getSeed(), getNegative(), useHierarchicSoftmax, getSampling(), getLearningRateDecayWords(), hugeModelExpected, useUnknown, getScavengerActivationThreshold(), getScavengerRetentionDelay(), getElementsLearningAlgorithm(), getSequenceLearningAlgorithm(), getModelUtils(), getTokenizerFactory(), getTokenPreProcessor(), getnGram(), getUNK(), getSTOP(), getStopList(), getVocabSize(), trainElementsVectors, trainSequenceVectors, allowParallelTokenization, preciseWeightInit, preciseMode);
+        result = 31 * result + Arrays.hashCode(getVariableWindows());
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "VectorsConfiguration{" +
+                "minWordFrequency=" + minWordFrequency +
+                ", learningRate=" + learningRate +
+                ", minLearningRate=" + minLearningRate +
+                ", layersSize=" + layersSize +
+                ", useAdaGrad=" + useAdaGrad +
+                ", batchSize=" + batchSize +
+                ", iterations=" + iterations +
+                ", epochs=" + epochs +
+                ", window=" + window +
+                ", seed=" + seed +
+                ", negative=" + negative +
+                ", useHierarchicSoftmax=" + useHierarchicSoftmax +
+                ", sampling=" + sampling +
+                ", learningRateDecayWords=" + learningRateDecayWords +
+                ", variableWindows=" + Arrays.toString(variableWindows) +
+                ", hugeModelExpected=" + hugeModelExpected +
+                ", useUnknown=" + useUnknown +
+                ", scavengerActivationThreshold=" + scavengerActivationThreshold +
+                ", scavengerRetentionDelay=" + scavengerRetentionDelay +
+                ", elementsLearningAlgorithm='" + elementsLearningAlgorithm + '\'' +
+                ", sequenceLearningAlgorithm='" + sequenceLearningAlgorithm + '\'' +
+                ", modelUtils='" + modelUtils + '\'' +
+                ", tokenizerFactory='" + tokenizerFactory + '\'' +
+                ", tokenPreProcessor='" + tokenPreProcessor + '\'' +
+                ", nGram=" + nGram +
+                ", UNK='" + UNK + '\'' +
+                ", STOP='" + STOP + '\'' +
+                ", stopList=" + stopList +
+                ", vocabSize=" + vocabSize +
+                ", trainElementsVectors=" + trainElementsVectors +
+                ", trainSequenceVectors=" + trainSequenceVectors +
+                ", allowParallelTokenization=" + allowParallelTokenization +
+                ", preciseWeightInit=" + preciseWeightInit +
+                ", preciseMode=" + preciseMode +
+                '}';
+    }
+
+    public static void setMapper(ObjectMapper mapper) {
+        VectorsConfiguration.mapper = mapper;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/ModelUtils.java

@@ -0,0 +1,105 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings.reader;
+
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.nd4j.linalg.api.ndarray.INDArray;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public interface ModelUtils<T extends SequenceElement> {
+
+    /**
+     * This method implementations should accept given lookup table, and use them in further calls to interface methods
+     *
+     * @param lookupTable
+     */
+    void init(WeightLookupTable<T> lookupTable);
+
+    /**
+     * This method implementations should return distance between two given elements
+     *
+     * @param label1
+     * @param label2
+     * @return
+     */
+    double similarity(String label1, String label2);
+
+
+    /**
+     * Accuracy based on questions which are a space separated list of strings
+     * where the first word is the query word, the next 2 words are negative,
+     * and the last word is the predicted word to be nearest
+     * @param questions the questions to ask
+     * @return the accuracy based on these questions
+     */
+    Map<String, Double> accuracy(List<String> questions);
+
+
+    /**
+     * Find all words with a similar characters
+     * in the vocab
+     * @param word the word to compare
+     * @param accuracy the accuracy: 0 to 1
+     * @return the list of words that are similar in the vocab
+     */
+    List<String> similarWordsInVocabTo(String word, double accuracy);
+
+
+    /**
+     * This method implementations should return N nearest elements labels to given element's label
+     *
+     * @param label label to return nearest elements for
+     * @param n number of nearest words to return
+     * @return
+     */
+    Collection<String> wordsNearest(String label, int n);
+
+    /**
+     * Words nearest based on positive and negative words
+     *
+     * @param positive the positive words
+     * @param negative the negative words
+     * @param top the top n words
+     * @return the words nearest the mean of the words
+     */
+    Collection<String> wordsNearest(Collection<String> positive, Collection<String> negative, int top);
+
+
+    /**
+     * Words nearest based on positive and negative words
+     * * @param top the top n words
+     * @return the words nearest the mean of the words
+     */
+    Collection<String> wordsNearest(INDArray words, int top);
+
+
+    Collection<String> wordsNearestSum(String word, int n);
+
+
+    Collection<String> wordsNearestSum(INDArray words, int top);
+
+    Collection<String> wordsNearestSum(Collection<String> positive, Collection<String> negative, int top);
+}
+

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/impl/BasicModelUtils.java

@@ -0,0 +1,453 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings.reader.impl;
+
+import org.nd4j.shade.guava.collect.Lists;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable;
+import org.deeplearning4j.models.embeddings.reader.ModelUtils;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.nd4j.common.util.MathUtils;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.ops.transforms.Transforms;
+import org.nd4j.common.primitives.Counter;
+import org.nd4j.common.util.SetUtils;
+
+import java.util.*;
+
+@Slf4j
+public class BasicModelUtils<T extends SequenceElement> implements ModelUtils<T> {
+    public static final String EXISTS = "exists";
+    public static final String CORRECT = "correct";
+    public static final String WRONG = "wrong";
+    protected volatile VocabCache<T> vocabCache;
+    protected volatile WeightLookupTable<T> lookupTable;
+
+    protected volatile boolean normalized = false;
+
+
+    public BasicModelUtils() {
+
+    }
+
+    @Override
+    public void init(@NonNull WeightLookupTable<T> lookupTable) {
+        this.vocabCache = lookupTable.getVocabCache();
+        this.lookupTable = lookupTable;
+
+        // reset normalization trigger on init call
+        this.normalized = false;
+    }
+
+    /**
+     * Returns the similarity of 2 words. Result value will be in range [-1,1], where -1.0 is exact opposite similarity, i.e. NO similarity, and 1.0 is total match of two word vectors.
+     * However, most of time you'll see values in range [0,1], but that's something depends of training corpus.
+     *
+     * Returns NaN if any of labels not exists in vocab, or any label is null
+     *
+     * @param label1 the first word
+     * @param label2 the second word
+     * @return a normalized similarity (cosine similarity)
+     */
+    @Override
+    public double similarity(@NonNull String label1, @NonNull String label2) {
+        if (label1 == null || label2 == null) {
+            log.debug("LABELS: " + label1 + ": " + (label1 == null ? "null" : EXISTS) + ";" + label2 + " vec2:"
+                            + (label2 == null ? "null" : EXISTS));
+            return Double.NaN;
+        }
+
+        if (!vocabCache.hasToken(label1)) {
+            log.debug("Unknown token 1 requested: [{}]", label1);
+            return Double.NaN;
+        }
+
+        if (!vocabCache.hasToken(label2)) {
+            log.debug("Unknown token 2 requested: [{}]", label2);
+            return Double.NaN;
+        }
+
+        INDArray vec1 = lookupTable.vector(label1).dup();
+        INDArray vec2 = lookupTable.vector(label2).dup();
+
+
+        if (vec1 == null || vec2 == null) {
+            log.debug(label1 + ": " + (vec1 == null ? "null" : EXISTS) + ";" + label2 + " vec2:"
+                            + (vec2 == null ? "null" : EXISTS));
+            return Double.NaN;
+        }
+
+        if (label1.equals(label2))
+            return 1.0;
+
+        return Transforms.cosineSim(vec1, vec2);
+    }
+
+
+    @Override
+    public Collection<String> wordsNearest(String label, int n) {
+        List<String> collection = new ArrayList<>(wordsNearest(Arrays.asList(label), new ArrayList<String>(), n + 1));
+        if (collection.contains(label))
+            collection.remove(label);
+
+        while (collection.size() > n)
+            collection.remove(collection.size() - 1);
+
+        return collection;
+    }
+
+    /**
+     * Accuracy based on questions which are a space separated list of strings
+     * where the first word is the query word, the next 2 words are negative,
+     * and the last word is the predicted word to be nearest
+     * @param questions the questions to ask
+     * @return the accuracy based on these questions
+     */
+    @Override
+    public Map<String, Double> accuracy(List<String> questions) {
+        Map<String, Double> accuracy = new HashMap<>();
+        Counter<String> right = new Counter<>();
+        String analogyType = "";
+        for (String s : questions) {
+            if (s.startsWith(":")) {
+                double correct = right.getCount(CORRECT);
+                double wrong = right.getCount(WRONG);
+                if (analogyType.isEmpty()) {
+                    analogyType = s;
+                    continue;
+                }
+                double accuracyRet = 100.0 * correct / (correct + wrong);
+                accuracy.put(analogyType, accuracyRet);
+                analogyType = s;
+                right.clear();
+            } else {
+                String[] split = s.split(" ");
+                List<String> positive = Arrays.asList(split[1], split[2]);
+                List<String> negative = Arrays.asList(split[0]);
+                String predicted = split[3];
+                String w = wordsNearest(positive, negative, 1).iterator().next();
+                if (predicted.equals(w))
+                    right.incrementCount(CORRECT, 1.0f);
+                else
+                    right.incrementCount(WRONG, 1.0f);
+
+            }
+        }
+        if (!analogyType.isEmpty()) {
+            double correct = right.getCount(CORRECT);
+            double wrong = right.getCount(WRONG);
+            double accuracyRet = 100.0 * correct / (correct + wrong);
+            accuracy.put(analogyType, accuracyRet);
+        }
+        return accuracy;
+    }
+
+    /**
+     * Find all words with a similar characters
+     * in the vocab
+     * @param word the word to compare
+     * @param accuracy the accuracy: 0 to 1
+     * @return the list of words that are similar in the vocab
+     */
+    @Override
+    public List<String> similarWordsInVocabTo(String word, double accuracy) {
+        List<String> ret = new ArrayList<>();
+        for (String s : vocabCache.words()) {
+            if (MathUtils.stringSimilarity(word, s) >= accuracy)
+                ret.add(s);
+        }
+        return ret;
+    }
+
+    public Collection<String> wordsNearest(@NonNull Collection<String> positive, @NonNull Collection<String> negative,
+                    int top) {
+        // Check every word is in the model
+        for (String p : SetUtils.union(new HashSet<>(positive), new HashSet<>(negative))) {
+            if (!vocabCache.containsWord(p)) {
+                return new ArrayList<>();
+            }
+        }
+
+        INDArray words = Nd4j.create(positive.size() + negative.size(), lookupTable.layerSize());
+        int row = 0;
+        //Set<String> union = SetUtils.union(new HashSet<>(positive), new HashSet<>(negative));
+        for (String s : positive) {
+            words.putRow(row++, lookupTable.vector(s));
+        }
+
+        for (String s : negative) {
+            words.putRow(row++, lookupTable.vector(s).mul(-1));
+        }
+
+        INDArray mean = words.isMatrix() ? words.mean(0).reshape(1, words.size(1)) : words;
+        Collection<String> tempRes = wordsNearest(mean, top + positive.size() + negative.size());
+        List<String> realResults = new ArrayList<>();
+
+        for (String word : tempRes) {
+            if (!positive.contains(word) && !negative.contains(word) && realResults.size() < top)
+                realResults.add(word);
+        }
+
+        return realResults;
+    }
+
+    /**
+     * Get the top n words most similar to the given word
+     * @param word the word to compare
+     * @param n the n to get
+     * @return the top n words
+     */
+    @Override
+    public Collection<String> wordsNearestSum(String word, int n) {
+        //INDArray vec = Transforms.unitVec(this.lookupTable.vector(word));
+        INDArray vec = this.lookupTable.vector(word);
+        return wordsNearestSum(vec, n);
+    }
+
+    protected INDArray adjustRank(INDArray words) {
+        if (lookupTable instanceof InMemoryLookupTable) {
+            InMemoryLookupTable l = (InMemoryLookupTable) lookupTable;
+
+            INDArray syn0 = l.getSyn0();
+            if (!words.dataType().equals(syn0.dataType())) {
+                return words.castTo(syn0.dataType());
+            }
+            if (words.rank() == 0 || words.rank() > 2) {
+                throw new IllegalStateException("Invalid rank for wordsNearest method");
+            } else if (words.rank() == 1) {
+                return words.reshape(1, -1);
+            }
+        }
+        return words;
+    }
+    /**
+     * Words nearest based on positive and negative words
+     * * @param top the top n words
+     * @return the words nearest the mean of the words
+     */
+    @Override
+    public Collection<String> wordsNearest(INDArray words, int top) {
+        words = adjustRank(words);
+
+        if (lookupTable instanceof InMemoryLookupTable) {
+            InMemoryLookupTable l = (InMemoryLookupTable) lookupTable;
+
+            INDArray syn0 = l.getSyn0();
+
+            if (!normalized) {
+                synchronized (this) {
+                    if (!normalized) {
+                        syn0.diviColumnVector(syn0.norm2(1));
+                        normalized = true;
+                    }
+                }
+            }
+
+            INDArray similarity = Transforms.unitVec(words).mmul(syn0.transpose());
+
+            List<Double> highToLowSimList = getTopN(similarity, top + 20);
+
+            List<WordSimilarity> result = new ArrayList<>();
+
+            for (int i = 0; i < highToLowSimList.size(); i++) {
+                String word = vocabCache.wordAtIndex(highToLowSimList.get(i).intValue());
+                if (word != null && !word.equals("UNK") && !word.equals("STOP")) {
+                    INDArray otherVec = lookupTable.vector(word);
+                    double sim = Transforms.cosineSim(words, otherVec);
+
+                    result.add(new WordSimilarity(word, sim));
+                }
+            }
+
+            Collections.sort(result, new SimilarityComparator());
+
+            return getLabels(result, top);
+        }
+
+        Counter<String> distances = new Counter<>();
+
+        for (String s : vocabCache.words()) {
+            INDArray otherVec = lookupTable.vector(s);
+            double sim = Transforms.cosineSim(words, otherVec);
+            distances.incrementCount(s, (float) sim);
+        }
+
+
+        distances.keepTopNElements(top);
+        return distances.keySet();
+
+
+    }
+
+    /**
+     * Get top N elements
+     *
+     * @param vec the vec to extract the top elements from
+     * @param N the number of elements to extract
+     * @return the indices and the sorted top N elements
+     */
+    private List<Double> getTopN(INDArray vec, int N) {
+        ArrayComparator comparator = new ArrayComparator();
+        PriorityQueue<Double[]> queue = new PriorityQueue<>(vec.rows(), comparator);
+
+        for (int j = 0; j < vec.length(); j++) {
+            final Double[] pair = new Double[] {vec.getDouble(j), (double) j};
+            if (queue.size() < N) {
+                queue.add(pair);
+            } else {
+                Double[] head = queue.peek();
+                if (comparator.compare(pair, head) > 0) {
+                    queue.poll();
+                    queue.add(pair);
+                }
+            }
+        }
+
+        List<Double> lowToHighSimLst = new ArrayList<>();
+
+        while (!queue.isEmpty()) {
+            double ind = queue.poll()[1];
+            lowToHighSimLst.add(ind);
+        }
+        return Lists.reverse(lowToHighSimLst);
+    }
+
+    /**
+     * Words nearest based on positive and negative words
+     * * @param top the top n words
+     * @return the words nearest the mean of the words
+     */
+    @Override
+    public Collection<String> wordsNearestSum(INDArray words, int top) {
+
+        if (lookupTable instanceof InMemoryLookupTable) {
+            InMemoryLookupTable l = (InMemoryLookupTable) lookupTable;
+            INDArray syn0 = l.getSyn0();
+            INDArray temp = syn0.norm2(0).rdivi(1).reshape(words.shape());
+            INDArray weights = temp.muli(words);
+            INDArray distances = syn0.mulRowVector(weights).sum(1);
+            INDArray[] sorted = Nd4j.sortWithIndices(distances, 0, false);
+            INDArray sort = sorted[0];
+            List<String> ret = new ArrayList<>();
+
+            if (top > sort.length())
+                top = (int) sort.length();
+            //there will be a redundant word
+            int end = top;
+            for (int i = 0; i < end; i++) {
+                String add = vocabCache.wordAtIndex(sort.getInt(i));
+                if (add == null || add.equals("UNK") || add.equals("STOP")) {
+                    end++;
+                    if (end >= sort.length())
+                        break;
+                    continue;
+                }
+                ret.add(vocabCache.wordAtIndex(sort.getInt(i)));
+            }
+            return ret;
+        }
+
+        Counter<String> distances = new Counter<>();
+
+        for (String s : vocabCache.words()) {
+            INDArray otherVec = lookupTable.vector(s);
+            double sim = Transforms.cosineSim(words, otherVec);
+            distances.incrementCount(s, (float) sim);
+        }
+
+        distances.keepTopNElements(top);
+        return distances.keySet();
+    }
+
+    /**
+     * Words nearest based on positive and negative words
+     * @param positive the positive words
+     * @param negative the negative words
+     * @param top the top n words
+     * @return the words nearest the mean of the words
+     */
+    @Override
+    public Collection<String> wordsNearestSum(Collection<String> positive, Collection<String> negative, int top) {
+        INDArray words = Nd4j.create(lookupTable.layerSize());
+        //    Set<String> union = SetUtils.union(new HashSet<>(positive), new HashSet<>(negative));
+        for (String s : positive)
+            words.addi(lookupTable.vector(s));
+
+
+        for (String s : negative)
+            words.addi(lookupTable.vector(s).mul(-1));
+
+        return wordsNearestSum(words, top);
+    }
+
+
+    public static class SimilarityComparator implements Comparator<WordSimilarity> {
+        @Override
+        public int compare(WordSimilarity o1, WordSimilarity o2) {
+            if (Double.isNaN(o1.getSimilarity()) && Double.isNaN(o2.getSimilarity())) {
+                return 0;
+            } else if (Double.isNaN(o1.getSimilarity()) && !Double.isNaN(o2.getSimilarity())) {
+                return -1;
+            } else if (!Double.isNaN(o1.getSimilarity()) && Double.isNaN(o2.getSimilarity())) {
+                return 1;
+            }
+            return Double.compare(o2.getSimilarity(), o1.getSimilarity());
+        }
+    }
+
+    public static class ArrayComparator implements Comparator<Double[]> {
+        @Override
+        public int compare(Double[] o1, Double[] o2) {
+            if (Double.isNaN(o1[0]) && Double.isNaN(o2[0])) {
+                return 0;
+            } else if (Double.isNaN(o1[0]) && !Double.isNaN(o2[0])) {
+                return -1;
+            } else if (!Double.isNaN(o1[0]) && Double.isNaN(o2[0])) {
+                return 1;
+            }
+            return Double.compare(o1[0], o2[0]);
+        }
+    }
+
+    @Data
+    @AllArgsConstructor
+    public static class WordSimilarity {
+        private String word;
+        private double similarity;
+    }
+
+    public static List<String> getLabels(List<WordSimilarity> results, int limit) {
+        List<String> result = new ArrayList<>();
+        for (int x = 0; x < results.size(); x++) {
+            result.add(results.get(x).getWord());
+            if (result.size() >= limit)
+                break;
+        }
+
+        return result;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/impl/FlatModelUtils.java

@@ -0,0 +1,75 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings.reader.impl;
+
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.ops.transforms.Transforms;
+import org.nd4j.common.primitives.Counter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+
+public class FlatModelUtils<T extends SequenceElement> extends BasicModelUtils<T> {
+    private static final Logger log = LoggerFactory.getLogger(FlatModelUtils.class);
+
+    public FlatModelUtils() {
+
+    }
+
+    /**
+     * This method does full scan against whole vocabulary, building descending list of similar words
+     * @param label
+     * @param n
+     * @return
+     */
+    @Override
+    public Collection<String> wordsNearest(String label, int n) {
+        Collection<String> collection = wordsNearest(lookupTable.vector(label), n);
+        if (collection.contains(label))
+            collection.remove(label);
+        return collection;
+    }
+
+    /**
+     * This method does full scan against whole vocabulary, building descending list of similar words
+     *
+     * @param words
+     * @param top
+     * @return the words nearest the mean of the words
+     */
+    @Override
+    public Collection<String> wordsNearest(INDArray words, int top) {
+        Counter<String> distances = new Counter<>();
+
+        words = adjustRank(words);
+
+        for (String s : vocabCache.words()) {
+            INDArray otherVec = lookupTable.vector(s);
+            double sim = Transforms.cosineSim(Transforms.unitVec(words.dup()), Transforms.unitVec(otherVec.dup()));
+            distances.incrementCount(s, (float) sim);
+        }
+
+        distances.keepTopNElements(top);
+        return distances.keySetSorted();
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectors.java

@@ -0,0 +1,179 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings.wordvectors;
+
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.embeddings.reader.ModelUtils;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.deeplearning4j.nn.weights.embeddings.EmbeddingInitializer;
+import org.nd4j.linalg.api.ndarray.INDArray;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public interface WordVectors extends Serializable, EmbeddingInitializer {
+
+    String getUNK();
+
+    void setUNK(String newUNK);
+
+    /**
+     * Returns true if the model has this word in the vocab
+     * @param word the word to test for
+     * @return true if the model has the word in the vocab
+     */
+    boolean hasWord(String word);
+
+    Collection<String> wordsNearest(INDArray words, int top);
+
+    Collection<String> wordsNearestSum(INDArray words, int top);
+
+    /**
+     * Get the top n words most similar to the given word
+     * @param word the word to compare
+     * @param n the n to get
+     * @return the top n words
+     */
+    Collection<String> wordsNearestSum(String word, int n);
+
+
+    /**
+     * Words nearest based on positive and negative words
+     * @param positive the positive words
+     * @param negative the negative words
+     * @param top the top n words
+     * @return the words nearest the mean of the words
+     */
+    Collection<String> wordsNearestSum(Collection<String> positive, Collection<String> negative, int top);
+
+    /**
+     * Accuracy based on questions which are a space separated list of strings
+     * where the first word is the query word, the next 2 words are negative,
+     * and the last word is the predicted word to be nearest
+     * @param questions the questions to ask
+     * @return the accuracy based on these questions
+     */
+    Map<String, Double> accuracy(List<String> questions);
+
+    int indexOf(String word);
+
+    /**
+     * Find all words with a similar characters
+     * in the vocab
+     * @param word the word to compare
+     * @param accuracy the accuracy: 0 to 1
+     * @return the list of words that are similar in the vocab
+     */
+    List<String> similarWordsInVocabTo(String word, double accuracy);
+
+    /**
+     * Get the word vector for a given matrix
+     * @param word the word to get the matrix for
+     * @return the ndarray for this word
+     */
+    double[] getWordVector(String word);
+
+    /**
+     * Returns the word vector divided by the norm2 of the array
+     * @param word the word to get the matrix for
+     * @return the looked up matrix
+     */
+    INDArray getWordVectorMatrixNormalized(String word);
+
+    /**
+     * Get the word vector for a given matrix
+     * @param word the word to get the matrix for
+     * @return the ndarray for this word
+     */
+    INDArray getWordVectorMatrix(String word);
+
+
+    /**
+     * This method returns 2D array, where each row represents corresponding word/label
+     *
+     * @param labels
+     * @return
+     */
+    INDArray getWordVectors(Collection<String> labels);
+
+    /**
+     * This method returns mean vector, built from words/labels passed in
+     *
+     * @param labels
+     * @return
+     */
+    INDArray getWordVectorsMean(Collection<String> labels);
+
+    /**
+     * Words nearest based on positive and negative words
+     * @param positive the positive words
+     * @param negative the negative words
+     * @param top the top n words
+     * @return the words nearest the mean of the words
+     */
+    Collection<String> wordsNearest(Collection<String> positive, Collection<String> negative, int top);
+
+
+    /**
+     * Get the top n words most similar to the given word
+     * @param word the word to compare
+     * @param n the n to get
+     * @return the top n words
+     */
+    Collection<String> wordsNearest(String word, int n);
+
+
+
+    /**
+     * Returns the similarity of 2 words
+     * @param word the first word
+     * @param word2 the second word
+     * @return a normalized similarity (cosine similarity)
+     */
+    double similarity(String word, String word2);
+
+    /**
+     * Vocab for the vectors
+     * @return
+     */
+    VocabCache vocab();
+
+    /**
+     * Lookup table for the vectors
+     * @return
+     */
+    WeightLookupTable lookupTable();
+
+    /**
+     * Specifies ModelUtils to be used to access model
+     * @param utils
+     */
+    void setModelUtils(ModelUtils utils);
+
+   /**
+     * Does implementation vectorize words absent in vocabulary
+     * @return boolean
+     */
+    boolean outOfVocabularySupported();
+
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectorsImpl.java

@@ -0,0 +1,373 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.embeddings.wordvectors;
+
+import org.nd4j.shade.guava.util.concurrent.AtomicDouble;
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+import org.apache.commons.lang.ArrayUtils;
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable;
+import org.deeplearning4j.models.embeddings.reader.ModelUtils;
+import org.deeplearning4j.models.embeddings.reader.impl.BasicModelUtils;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.factory.Nd4j;
+
+
+import java.util.*;
+
+public class WordVectorsImpl<T extends SequenceElement> implements WordVectors {
+    private static final long serialVersionUID = 78249242142L;
+
+    //number of times the word must occur in the vocab to appear in the calculations, otherwise treat as unknown
+    @Getter
+    protected int minWordFrequency = 5;
+    @Getter
+    protected WeightLookupTable<T> lookupTable;
+    @Getter
+    protected VocabCache<T> vocab;
+    protected int layerSize = 100;
+    @Getter
+    protected transient ModelUtils<T> modelUtils = new BasicModelUtils<>();
+    private boolean initDone = false;
+    @Getter
+    @Setter
+    protected int numIterations = 1;
+    @Getter
+    @Setter
+    protected int numEpochs = 1;
+    @Getter
+    @Setter
+    protected double negative = 0;
+    @Getter
+    @Setter
+    protected double sampling = 0;
+    protected AtomicDouble learningRate = new AtomicDouble(0.025);
+    @Getter
+    @Setter
+    protected double minLearningRate = 0.01;
+    @Getter
+    @Setter
+    protected int window = 5;
+    @Getter
+    @Setter
+    protected int batchSize;
+    @Getter
+    @Setter
+    protected int learningRateDecayWords;
+    @Getter
+    @Setter
+    protected boolean resetModel;
+    protected boolean useAdeGrad;
+    @Getter
+    @Setter
+    protected int workers = 1;
+
+    @Getter
+    @Setter
+    protected int vectorCalcThreads = 1;
+
+    @Getter
+    @Setter
+    protected boolean trainSequenceVectors = false;
+    @Getter
+    @Setter
+    protected boolean trainElementsVectors = true;
+    @Getter
+    @Setter
+    protected long seed;
+    @Getter
+    @Setter
+    protected boolean useUnknown = false;
+    @Getter
+    @Setter
+    protected int[] variableWindows;
+
+
+    /**
+     * This method returns word vector size
+     *
+     * @return
+     */
+    public int getLayerSize() {
+        if (lookupTable != null && lookupTable.getWeights() != null) {
+            return lookupTable.getWeights().columns();
+        } else
+            return layerSize;
+    }
+
+    public final static String DEFAULT_UNK = "UNK";
+    @Getter
+    @Setter
+    private String UNK = DEFAULT_UNK;
+
+    @Getter
+    protected Collection<String> stopWords = new ArrayList<>(); //StopWords.getStopWords();
+
+    /**
+     * Returns true if the model has this word in the vocab
+     * @param word the word to test for
+     * @return true if the model has the word in the vocab
+     */
+    public boolean hasWord(String word) {
+        return vocab().indexOf(word) >= 0;
+    }
+
+    /**
+     * Words nearest based on positive and negative words
+     * @param positive the positive words
+     * @param negative the negative words
+     * @param top the top n words
+     * @return the words nearest the mean of the words
+     */
+    public Collection<String> wordsNearestSum(Collection<String> positive, Collection<String> negative, int top) {
+        return modelUtils.wordsNearestSum(positive, negative, top);
+    }
+
+    /**
+     * Words nearest based on positive and negative words
+     * * @param top the top n words
+     * @return the words nearest the mean of the words
+     */
+    @Override
+    public Collection<String> wordsNearestSum(INDArray words, int top) {
+        return modelUtils.wordsNearestSum(words, top);
+    }
+
+    /**
+     * Words nearest based on positive and negative words
+     * * @param top the top n words
+     * @return the words nearest the mean of the words
+     */
+    @Override
+    public Collection<String> wordsNearest(INDArray words, int top) {
+        return modelUtils.wordsNearest(words, top);
+    }
+
+    /**
+     * Get the top n words most similar to the given word
+     * @param word the word to compare
+     * @param n the n to get
+     * @return the top n words
+     */
+    public Collection<String> wordsNearestSum(String word, int n) {
+        return modelUtils.wordsNearestSum(word, n);
+    }
+
+
+    /**
+     * Accuracy based on questions which are a space separated list of strings
+    * where the first word is the query word, the next 2 words are negative,
+    * and the last word is the predicted word to be nearest
+    * @param questions the questions to ask
+    * @return the accuracy based on these questions
+    */
+    public Map<String, Double> accuracy(List<String> questions) {
+        return modelUtils.accuracy(questions);
+    }
+
+    @Override
+    public int indexOf(String word) {
+        return vocab().indexOf(word);
+    }
+
+
+    /**
+     * Find all words with a similar characters
+     * in the vocab
+     * @param word the word to compare
+     * @param accuracy the accuracy: 0 to 1
+     * @return the list of words that are similar in the vocab
+     */
+    public List<String> similarWordsInVocabTo(String word, double accuracy) {
+        return this.modelUtils.similarWordsInVocabTo(word, accuracy);
+    }
+
+    /**
+     * Get the word vector for a given matrix
+     * @param word the word to get the matrix for
+     * @return the ndarray for this word
+     */
+    public double[] getWordVector(String word) {
+        INDArray r = getWordVectorMatrix(word);
+        if (r == null)
+            return null;
+        return r.dup().data().asDouble();
+    }
+
+    /**
+     * Returns the word vector divided by the norm2 of the array
+     * @param word the word to get the matrix for
+     * @return the looked up matrix
+     */
+    public INDArray getWordVectorMatrixNormalized(String word) {
+        INDArray r = getWordVectorMatrix(word);
+        if (r == null)
+            return null;
+
+        return r.div(Nd4j.getBlasWrapper().nrm2(r));
+    }
+
+    @Override
+    public INDArray getWordVectorMatrix(String word) {
+        return lookupTable().vector(word);
+    }
+
+
+    /**
+     * Words nearest based on positive and negative words
+     *
+     * @param positive the positive words
+     * @param negative the negative words
+     * @param top the top n words
+     * @return the words nearest the mean of the words
+     */
+    @Override
+    public Collection<String> wordsNearest(Collection<String> positive, Collection<String> negative, int top) {
+        return modelUtils.wordsNearest(positive, negative, top);
+    }
+
+    /**
+     * This method returns 2D array, where each row represents corresponding label
+     *
+     * @param labels
+     * @return
+     */
+    @Override
+    public INDArray getWordVectors(@NonNull Collection<String> labels) {
+        int indexes[] = new int[labels.size()];
+        int cnt = 0;
+        boolean useIndexUnknown = useUnknown && vocab.containsWord(getUNK());
+
+        for (String label : labels) {
+            if (vocab.containsWord(label)) {
+                indexes[cnt] = vocab.indexOf(label);
+            } else
+                indexes[cnt] = useIndexUnknown ? vocab.indexOf(getUNK()) : -1;
+            cnt++;
+        }
+
+        while (ArrayUtils.contains(indexes, -1)) {
+            indexes = ArrayUtils.removeElement(indexes, -1);
+        }
+        if (indexes.length == 0) {
+                return Nd4j.empty(((InMemoryLookupTable)lookupTable).getSyn0().dataType());
+        }
+
+        INDArray result = Nd4j.pullRows(lookupTable.getWeights(), 1, indexes);
+        return result;
+    }
+
+    /**
+     * This method returns mean vector, built from words/labels passed in
+     *
+     * @param labels
+     * @return
+     */
+    @Override
+    public INDArray getWordVectorsMean(Collection<String> labels) {
+        INDArray array = getWordVectors(labels);
+        return array.mean(0);
+    }
+
+    /**
+     * Get the top n words most similar to the given word
+     * @param word the word to compare
+     * @param n the n to get
+     * @return the top n words
+     */
+    public Collection<String> wordsNearest(String word, int n) {
+        return modelUtils.wordsNearest(word, n);
+    }
+
+
+    /**
+     * Returns similarity of two elements, provided by ModelUtils
+     *
+     * @param word the first word
+     * @param word2 the second word
+     * @return a normalized similarity (cosine similarity)
+     */
+    public double similarity(String word, String word2) {
+        return modelUtils.similarity(word, word2);
+    }
+
+    @Override
+    public VocabCache<T> vocab() {
+        return vocab;
+    }
+
+    @Override
+    public WeightLookupTable lookupTable() {
+        return lookupTable;
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public void setModelUtils(@NonNull ModelUtils modelUtils) {
+        if (lookupTable != null) {
+            modelUtils.init(lookupTable);
+            this.modelUtils = modelUtils;
+            //0.25, -0.03, -0.47, 0.10, -0.25, 0.28, 0.37,
+        }
+    }
+
+    public void setLookupTable(@NonNull WeightLookupTable lookupTable) {
+        this.lookupTable = lookupTable;
+        if (modelUtils == null)
+            this.modelUtils = new BasicModelUtils<>();
+
+        this.modelUtils.init(lookupTable);
+    }
+
+    public void setVocab(VocabCache vocab) {
+        this.vocab = vocab;
+    }
+
+
+    @Override
+    public void loadWeightsInto(INDArray array) {
+        array.assign(lookupTable.getWeights());
+    }
+
+    @Override
+    public long vocabSize() {
+        return lookupTable.getWeights().size(0);
+    }
+
+    @Override
+    public int vectorSize() {
+        return lookupTable.layerSize();
+    }
+
+    @Override
+    public boolean jsonSerializable() {
+        return false;
+    }
+
+    @Override
+    public boolean outOfVocabularySupported() {
+        return false;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/fasttext/FTLossFunctions.java

@@ -0,0 +1,38 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.fasttext;
+
+public enum FTLossFunctions {
+    HS("hs"),
+    NS("ns"),
+    SOFTMAX("softmax");
+
+    private final String name;
+
+    FTLossFunctions(String name) {
+        this.name = name;
+    }
+
+    @Override
+    public String toString() {
+        return this.name;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/fasttext/FTModels.java

@@ -0,0 +1,38 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.fasttext;
+
+public enum FTModels {
+    CBOW("cbow"),
+    SG("sg"),
+    SUP("sup");
+
+    private final String name;
+
+    FTModels(String name) {
+        this.name = name;
+    }
+
+    @Override
+    public String toString() {
+        return this.name;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/fasttext/FTOptions.java

@@ -0,0 +1,37 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.fasttext;
+
+public enum FTOptions {
+    INPUT_FILE("-input"),
+    OUTPUT_FILE("-output");
+
+    private final String name;
+
+    FTOptions(String name) {
+        this.name = name;
+    }
+
+    @Override
+    public String toString() {
+        return this.name;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/fasttext/FastText.java

@@ -0,0 +1,545 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.fasttext;
+
+import com.github.jfasttext.JFastText;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang3.NotImplementedException;
+import org.apache.commons.lang3.StringUtils;
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer;
+import org.deeplearning4j.models.embeddings.reader.ModelUtils;
+import org.deeplearning4j.models.embeddings.wordvectors.WordVectors;
+import org.deeplearning4j.models.word2vec.VocabWord;
+import org.deeplearning4j.models.word2vec.Word2Vec;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.deeplearning4j.models.word2vec.wordstore.inmemory.AbstractCache;
+import org.deeplearning4j.text.sentenceiterator.SentenceIterator;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.common.primitives.Pair;
+
+import java.io.*;
+import java.util.*;
+
+@Slf4j
+@AllArgsConstructor
+@lombok.Builder
+public class FastText implements WordVectors, Serializable {
+
+    private final static String METHOD_NOT_AVAILABLE = "This method is available for text (.vec) models only - binary (.bin) model currently loaded";
+    // Mandatory
+    @Getter private String inputFile;
+    @Getter private String outputFile;
+
+    // Optional for dictionary
+    @Builder.Default  private int bucket = -1;
+    @Builder.Default  private int minCount = -1;
+    @Builder.Default  private int minCountLabel = -1;
+    @Builder.Default  private int wordNgrams = -1;
+    @Builder.Default  private int minNgramLength = -1;
+    @Builder.Default  private int maxNgramLength = -1;
+    @Builder.Default  private int samplingThreshold = -1;
+    private String labelPrefix;
+
+    // Optional for training
+    @Getter private boolean supervised;
+    @Getter private boolean quantize;
+    @Getter private boolean predict;
+    @Getter private boolean predict_prob;
+    @Getter private boolean skipgram;
+    @Getter private boolean cbow;
+    @Getter private boolean nn;
+    @Getter private boolean analogies;
+    @Getter private String pretrainedVectorsFile;
+    @Getter
+    @Builder.Default
+    private double learningRate = -1.0;
+    @Getter private double learningRateUpdate = -1.0;
+    @Getter
+    @Builder.Default
+    private int dim = -1;
+    @Getter
+    @Builder.Default
+    private int contextWindowSize = -1;
+    @Getter
+    @Builder.Default
+    private int epochs = -1;
+    @Getter private String modelName;
+    @Getter private String lossName;
+    @Getter
+    @Builder.Default
+    private int negativeSamples = -1;
+    @Getter
+    @Builder.Default
+    private int numThreads = -1;
+    @Getter private boolean saveOutput = false;
+
+    // Optional for quantization
+    @Getter
+    @Builder.Default
+    private int cutOff = -1;
+    @Getter private boolean retrain;
+    @Getter private boolean qnorm;
+    @Getter private boolean qout;
+    @Getter
+    @Builder.Default
+    private int dsub = -1;
+
+    @Getter private SentenceIterator iterator;
+
+    @Builder.Default private transient JFastText fastTextImpl = new JFastText();
+    private transient Word2Vec word2Vec;
+    @Getter private boolean modelLoaded;
+    @Getter private boolean modelVectorsLoaded;
+    private VocabCache vocabCache;
+
+    public FastText(File modelPath) {
+        this();
+        loadBinaryModel(modelPath.getAbsolutePath());
+    }
+
+    public FastText() {
+        fastTextImpl = new JFastText();
+    }
+
+    private static class ArgsFactory {
+
+        private List<String> args = new ArrayList<>();
+
+        private void add(String label, String value) {
+            args.add(label);
+            args.add(value);
+        }
+
+        private void addOptional(String label, int value) {
+            if (value >= 0) {
+                args.add(label);
+                args.add(Integer.toString(value));
+            }
+        }
+
+        private void addOptional(String label, double value) {
+            if (value >= 0.0) {
+                args.add(label);
+                args.add(Double.toString(value));
+            }
+        }
+
+        private void addOptional(String label, String value) {
+            if (StringUtils.isNotEmpty(value)) {
+                args.add(label);
+                args.add(value);
+            }
+        }
+
+        private void addOptional(String label, boolean value) {
+            if (value) {
+                args.add(label);
+            }
+        }
+
+
+        public String[] args() {
+            String[] asArray = new String[args.size()];
+            return args.toArray(asArray);
+        }
+    }
+
+    private String[] makeArgs() {
+        ArgsFactory argsFactory = new ArgsFactory();
+
+        argsFactory.addOptional("cbow", cbow);
+        argsFactory.addOptional("skipgram", skipgram);
+        argsFactory.addOptional("supervised", supervised);
+        argsFactory.addOptional("quantize", quantize);
+        argsFactory.addOptional("predict", predict);
+        argsFactory.addOptional("predict_prob", predict_prob);
+
+        argsFactory.add("-input", inputFile);
+        argsFactory.add("-output", outputFile );
+
+        argsFactory.addOptional("-pretrainedVectors", pretrainedVectorsFile);
+
+        argsFactory.addOptional("-bucket", bucket);
+        argsFactory.addOptional("-minCount", minCount);
+        argsFactory.addOptional("-minCountLabel", minCountLabel);
+        argsFactory.addOptional("-wordNgrams", wordNgrams);
+        argsFactory.addOptional("-minn", minNgramLength);
+        argsFactory.addOptional("-maxn", maxNgramLength);
+        argsFactory.addOptional("-t", samplingThreshold);
+        argsFactory.addOptional("-label", labelPrefix);
+        argsFactory.addOptional("analogies",analogies);
+        argsFactory.addOptional("-lr", learningRate);
+        argsFactory.addOptional("-lrUpdateRate", learningRateUpdate);
+        argsFactory.addOptional("-dim", dim);
+        argsFactory.addOptional("-ws", contextWindowSize);
+        argsFactory.addOptional("-epoch", epochs);
+        argsFactory.addOptional("-loss", lossName);
+        argsFactory.addOptional("-neg", negativeSamples);
+        argsFactory.addOptional("-thread", numThreads);
+        argsFactory.addOptional("-saveOutput", saveOutput);
+        argsFactory.addOptional("-cutoff", cutOff);
+        argsFactory.addOptional("-retrain", retrain);
+        argsFactory.addOptional("-qnorm", qnorm);
+        argsFactory.addOptional("-qout", qout);
+        argsFactory.addOptional("-dsub", dsub);
+
+        return argsFactory.args();
+    }
+
+    public void fit() {
+        String[] cmd = makeArgs();
+        fastTextImpl.runCmd(cmd);
+    }
+
+    public void loadIterator() {
+        if (iterator != null) {
+            try {
+                File tempFile = File.createTempFile("FTX", ".txt");
+                BufferedWriter writer = new BufferedWriter(new FileWriter(tempFile));
+                while (iterator.hasNext()) {
+                    String sentence = iterator.nextSentence();
+                    writer.write(sentence);
+                }
+
+                fastTextImpl = new JFastText();
+            } catch (IOException e) {
+                log.error(e.getMessage());
+            }
+        }
+    }
+
+    public void loadPretrainedVectors(File vectorsFile) {
+        word2Vec = WordVectorSerializer.readWord2VecModel(vectorsFile);
+        modelVectorsLoaded = true;
+        log.info("Loaded vectorized representation from file %s. Functionality will be restricted.",
+                vectorsFile.getAbsolutePath());
+    }
+
+    public void loadBinaryModel(String modelPath) {
+        fastTextImpl.loadModel(modelPath);
+
+        modelLoaded = true;
+    }
+
+    public void unloadBinaryModel() {
+        fastTextImpl.unloadModel();
+        modelLoaded = false;
+    }
+
+    public void test(File testFile) {
+        fastTextImpl.test(testFile.getAbsolutePath());
+    }
+
+    private void assertModelLoaded() {
+        if (!modelLoaded && !modelVectorsLoaded)
+            throw new IllegalStateException("Model must be loaded before predict!");
+    }
+
+    public String predict(String text) {
+
+        assertModelLoaded();
+
+        String label = fastTextImpl.predict(text);
+        return label;
+    }
+
+    public Pair<String, Float> predictProbability(String text) {
+
+        assertModelLoaded();
+
+        JFastText.ProbLabel predictedProbLabel = fastTextImpl.predictProba(text);
+
+        Pair<String,Float> retVal = new Pair<>();
+        retVal.setFirst(predictedProbLabel.label);
+        retVal.setSecond(predictedProbLabel.logProb);
+        return retVal;
+    }
+
+    @Override
+    public VocabCache vocab() {
+        if (modelVectorsLoaded) {
+            vocabCache = word2Vec.vocab();
+        }
+        else {
+            if (!modelLoaded)
+                throw new IllegalStateException("Load model before calling vocab()");
+
+            if (vocabCache == null) {
+                vocabCache = new AbstractCache();
+            }
+            List<String> words = fastTextImpl.getWords();
+            for (int i = 0; i < words.size(); ++i) {
+                vocabCache.addWordToIndex(i, words.get(i));
+                VocabWord word = new VocabWord();
+                word.setWord(words.get(i));
+                vocabCache.addToken(word);
+            }
+        }
+        return vocabCache;
+    }
+
+    @Override
+    public long vocabSize() {
+        long result = 0;
+        if (modelVectorsLoaded) {
+            result = word2Vec.vocabSize();
+        }
+        else {
+            if (!modelLoaded)
+                throw new IllegalStateException("Load model before calling vocab()");
+            result = fastTextImpl.getNWords();
+        }
+        return result;
+    }
+
+    @Override
+    public String getUNK() {
+        throw new NotImplementedException("FastText.getUNK");
+    }
+
+    @Override
+    public void setUNK(String input) {
+        throw new NotImplementedException("FastText.setUNK");
+    }
+
+    @Override
+    public double[] getWordVector(String word) {
+        if (modelVectorsLoaded) {
+            return word2Vec.getWordVector(word);
+        }
+        else {
+            List<Float> vectors = fastTextImpl.getVector(word);
+            double[] retVal = new double[vectors.size()];
+            for (int i = 0; i < vectors.size(); ++i) {
+                retVal[i] = vectors.get(i);
+            }
+            return retVal;
+        }
+    }
+
+    @Override
+    public INDArray getWordVectorMatrixNormalized(String word) {
+        if (modelVectorsLoaded) {
+            return word2Vec.getWordVectorMatrixNormalized(word);
+        }
+        else {
+            INDArray r = getWordVectorMatrix(word);
+            return r.divi(Nd4j.getBlasWrapper().nrm2(r));
+        }
+    }
+
+    @Override
+    public INDArray getWordVectorMatrix(String word) {
+        if (modelVectorsLoaded) {
+            return word2Vec.getWordVectorMatrix(word);
+        }
+        else {
+            double[] values = getWordVector(word);
+            return Nd4j.createFromArray(values);
+        }
+    }
+
+    @Override
+    public INDArray getWordVectors(Collection<String> labels) {
+        if (modelVectorsLoaded) {
+            return word2Vec.getWordVectors(labels);
+        }
+        return null;
+    }
+
+    @Override
+    public INDArray getWordVectorsMean(Collection<String> labels) {
+        if (modelVectorsLoaded) {
+            return word2Vec.getWordVectorsMean(labels);
+        }
+        return null;
+    }
+
+    private List<String> words = new ArrayList<>();
+
+    @Override
+    public boolean hasWord(String word) {
+        if (modelVectorsLoaded) {
+            return word2Vec.outOfVocabularySupported();
+        }
+        if (words.isEmpty())
+            words = fastTextImpl.getWords();
+        return words.contains(word);
+    }
+
+    @Override
+    public Collection<String> wordsNearest(INDArray words, int top) {
+        if (modelVectorsLoaded) {
+            return word2Vec.wordsNearest(words, top);
+        }
+        throw new IllegalStateException(METHOD_NOT_AVAILABLE);
+    }
+
+    @Override
+    public Collection<String> wordsNearestSum(INDArray words, int top) {
+        if (modelVectorsLoaded) {
+            return word2Vec.wordsNearestSum(words, top);
+        }
+        throw new IllegalStateException(METHOD_NOT_AVAILABLE);
+    }
+
+    @Override
+    public Collection<String> wordsNearestSum(String word, int n) {
+        if (modelVectorsLoaded) {
+            return word2Vec.wordsNearestSum(word, n);
+        }
+        throw new IllegalStateException(METHOD_NOT_AVAILABLE);
+    }
+
+
+    @Override
+    public Collection<String> wordsNearestSum(Collection<String> positive, Collection<String> negative, int top) {
+        if (modelVectorsLoaded) {
+            return word2Vec.wordsNearestSum(positive, negative, top);
+        }
+        throw new IllegalStateException(METHOD_NOT_AVAILABLE);
+    }
+
+    @Override
+    public Map<String, Double> accuracy(List<String> questions) {
+        if (modelVectorsLoaded) {
+            return word2Vec.accuracy(questions);
+        }
+        throw new IllegalStateException(METHOD_NOT_AVAILABLE);
+    }
+
+    @Override
+    public int indexOf(String word) {
+        if (modelVectorsLoaded) {
+            return word2Vec.indexOf(word);
+        }
+        return vocab().indexOf(word);
+    }
+
+
+    @Override
+    public List<String> similarWordsInVocabTo(String word, double accuracy) {
+        if (modelVectorsLoaded) {
+            return word2Vec.similarWordsInVocabTo(word, accuracy);
+        }
+        throw new IllegalStateException(METHOD_NOT_AVAILABLE);
+    }
+
+    @Override
+    public Collection<String> wordsNearest(Collection<String> positive, Collection<String> negative, int top) {
+        if (modelVectorsLoaded) {
+            return word2Vec.wordsNearest(positive, negative, top);
+        }
+        throw new IllegalStateException(METHOD_NOT_AVAILABLE);
+    }
+
+
+    @Override
+    public Collection<String> wordsNearest(String word, int n) {
+        if (modelVectorsLoaded) {
+            return word2Vec.wordsNearest(word,n);
+        }
+        throw new IllegalStateException(METHOD_NOT_AVAILABLE);
+    }
+
+
+    @Override
+    public double similarity(String word, String word2) {
+        if (modelVectorsLoaded) {
+            return word2Vec.similarity(word, word2);
+        }
+        throw new IllegalStateException(METHOD_NOT_AVAILABLE);
+    }
+
+    @Override
+    public WeightLookupTable lookupTable() {
+        if (modelVectorsLoaded) {
+            return word2Vec.lookupTable();
+        }
+        return null;
+    }
+
+    @Override
+    public void setModelUtils(ModelUtils utils) {
+    }
+
+    @Override
+    public void loadWeightsInto(INDArray array) {}
+
+    @Override
+    public int vectorSize() {return -1;}
+
+    @Override
+    public boolean jsonSerializable() {return false;}
+
+    public double getLearningRate() {
+        return fastTextImpl.getLr();
+    }
+
+    public int getDimension() {
+        return fastTextImpl.getDim();
+    }
+
+    public int getContextWindowSize() {
+        return fastTextImpl.getContextWindowSize();
+    }
+
+    public int getEpoch() {
+        return fastTextImpl.getEpoch();
+    }
+
+    public int getNegativesNumber() {
+        return fastTextImpl.getNSampledNegatives();
+    }
+
+    public int getWordNgrams() {
+        return fastTextImpl.getWordNgrams();
+    }
+
+    public String getLossName() {
+        return fastTextImpl.getLossName();
+    }
+
+    public String getModelName() {
+        return fastTextImpl.getModelName();
+    }
+
+    public int getNumberOfBuckets() {
+        return fastTextImpl.getBucket();
+    }
+
+    public String getLabelPrefix() {
+        return fastTextImpl.getLabelPrefix();
+    }
+
+    @Override
+    public boolean outOfVocabularySupported() {
+        return true;
+    }
+
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/node2vec/Node2Vec.java

@@ -0,0 +1,293 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.node2vec;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.embeddings.learning.ElementsLearningAlgorithm;
+import org.deeplearning4j.models.embeddings.learning.SequenceLearningAlgorithm;
+import org.deeplearning4j.models.embeddings.loader.VectorsConfiguration;
+import org.deeplearning4j.models.embeddings.reader.ModelUtils;
+import org.deeplearning4j.models.embeddings.wordvectors.WordVectors;
+import org.deeplearning4j.models.paragraphvectors.ParagraphVectors;
+import org.deeplearning4j.models.sequencevectors.SequenceVectors;
+import org.deeplearning4j.models.sequencevectors.graph.primitives.Vertex;
+import org.deeplearning4j.models.sequencevectors.graph.walkers.GraphWalker;
+import org.deeplearning4j.models.sequencevectors.interfaces.SequenceIterator;
+import org.deeplearning4j.models.sequencevectors.interfaces.VectorsListener;
+import org.deeplearning4j.models.sequencevectors.iterators.AbstractSequenceIterator;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.sequencevectors.transformers.impl.GraphTransformer;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.nd4j.linalg.api.ndarray.INDArray;
+
+import java.util.Collection;
+import java.util.List;
+
+@Slf4j
+@Deprecated
+public class Node2Vec<V extends SequenceElement, E extends Number> extends SequenceVectors<V> {
+
+    public INDArray inferVector(@NonNull Collection<Vertex<V>> vertices) {
+        return null;
+    }
+
+    public static class Builder<V extends SequenceElement, E extends Number> extends SequenceVectors.Builder<V> {
+        private GraphWalker<V> walker;
+
+        public Builder(@NonNull GraphWalker<V> walker, @NonNull VectorsConfiguration configuration) {
+            this.walker = walker;
+            this.configuration = configuration;
+
+            // FIXME: this will cause transformer initialization
+            GraphTransformer<V> transformer = new GraphTransformer.Builder<>(walker.getSourceGraph())
+                            .setGraphWalker(walker).shuffleOnReset(true).build();
+
+            this.iterator = new AbstractSequenceIterator.Builder<V>(transformer).build();
+        }
+
+
+        @Override
+        protected Builder<V, E> useExistingWordVectors(@NonNull WordVectors vec) {
+            super.useExistingWordVectors(vec);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> iterate(@NonNull SequenceIterator<V> iterator) {
+            super.iterate(iterator);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> sequenceLearningAlgorithm(@NonNull String algoName) {
+            super.sequenceLearningAlgorithm(algoName);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> sequenceLearningAlgorithm(@NonNull SequenceLearningAlgorithm<V> algorithm) {
+            super.sequenceLearningAlgorithm(algorithm);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> elementsLearningAlgorithm(@NonNull String algoName) {
+            super.elementsLearningAlgorithm(algoName);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> elementsLearningAlgorithm(@NonNull ElementsLearningAlgorithm<V> algorithm) {
+            super.elementsLearningAlgorithm(algorithm);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> iterations(int iterations) {
+            super.iterations(iterations);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> epochs(int numEpochs) {
+            super.epochs(numEpochs);
+            return this;
+        }
+
+        /**
+         * Sets number of threads running calculations.
+         * Note this is different from workers which affect
+         * the number of threads used to compute updates.
+         * This should be balanced with the number of workers.
+         * High number of threads will actually hinder performance.
+         *
+         * @param vectorCalcThreads the number of threads to compute updates
+         * @return
+         */
+        public Builder<V, E> vectorCalcThreads(int vectorCalcThreads) {
+            super.vectorCalcThreads(vectorCalcThreads);
+            return this;
+        }
+        @Override
+        public Builder<V, E> workers(int numWorkers) {
+            super.workers(numWorkers);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> useHierarchicSoftmax(boolean reallyUse) {
+            super.useHierarchicSoftmax(reallyUse);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> useAdaGrad(boolean reallyUse) {
+            super.useAdaGrad(reallyUse);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> layerSize(int layerSize) {
+            super.layerSize(layerSize);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> learningRate(double learningRate) {
+            super.learningRate(learningRate);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> minWordFrequency(int minWordFrequency) {
+            super.minWordFrequency(minWordFrequency);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> minLearningRate(double minLearningRate) {
+            super.minLearningRate(minLearningRate);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> resetModel(boolean reallyReset) {
+            super.resetModel(reallyReset);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> vocabCache(@NonNull VocabCache<V> vocabCache) {
+            super.vocabCache(vocabCache);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> lookupTable(@NonNull WeightLookupTable<V> lookupTable) {
+            super.lookupTable(lookupTable);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> sampling(double sampling) {
+            super.sampling(sampling);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> negativeSample(double negative) {
+            super.negativeSample(negative);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> stopWords(@NonNull List<String> stopList) {
+            super.stopWords(stopList);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> trainElementsRepresentation(boolean trainElements) {
+            super.trainElementsRepresentation(trainElements);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> trainSequencesRepresentation(boolean trainSequences) {
+            super.trainSequencesRepresentation(trainSequences);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> stopWords(@NonNull Collection<V> stopList) {
+            super.stopWords(stopList);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> windowSize(int windowSize) {
+            super.windowSize(windowSize);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> seed(long randomSeed) {
+            super.seed(randomSeed);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> modelUtils(@NonNull ModelUtils<V> modelUtils) {
+            super.modelUtils(modelUtils);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> useUnknown(boolean reallyUse) {
+            super.useUnknown(reallyUse);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> unknownElement(@NonNull V element) {
+            super.unknownElement(element);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> useVariableWindow(int... windows) {
+            super.useVariableWindow(windows);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> usePreciseWeightInit(boolean reallyUse) {
+            super.usePreciseWeightInit(reallyUse);
+            return this;
+        }
+
+        @Override
+        protected void presetTables() {
+            super.presetTables();
+        }
+
+        @Override
+        public Builder<V, E> setVectorsListeners(@NonNull Collection<VectorsListener<V>> vectorsListeners) {
+            super.setVectorsListeners(vectorsListeners);
+            return this;
+        }
+
+        @Override
+        public Builder<V, E> enableScavenger(boolean reallyEnable) {
+            super.enableScavenger(reallyEnable);
+            return this;
+        }
+
+        public Node2Vec<V, E> build() {
+            Node2Vec<V, E> node2vec = new Node2Vec<>();
+            node2vec.iterator = this.iterator;
+
+            return node2vec;
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/Consumer.java

@@ -0,0 +1,24 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors;
+
+public class Consumer {
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/PCService.java

@@ -0,0 +1,24 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors;
+
+public class PCService {
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/enums/ListenerEvent.java

@@ -0,0 +1,25 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.enums;
+
+public enum ListenerEvent {
+    EPOCH, ITERATION, LINE,
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/NoEdgeHandling.java

@@ -0,0 +1,25 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.enums;
+
+public enum NoEdgeHandling {
+    SELF_LOOP_ON_DISCONNECTED, EXCEPTION_ON_DISCONNECTED, PADDING_ON_DISCONNECTED, CUTOFF_ON_DISCONNECTED, RESTART_ON_DISCONNECTED,
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/PopularityMode.java

@@ -0,0 +1,25 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.enums;
+
+public enum PopularityMode {
+    MAXIMUM, AVERAGE, MINIMUM,
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/SamplingMode.java

@@ -0,0 +1,25 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.enums;
+
+public enum SamplingMode {
+    MAX_POPULARITY, MIN_POPULARITY, MEDIAN_POPULARITY, RANDOM,
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/SpreadSpectrum.java

@@ -0,0 +1,25 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.enums;
+
+public enum SpreadSpectrum {
+    PLAIN, PROPORTIONAL,
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/WalkDirection.java

@@ -0,0 +1,25 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.enums;
+
+public enum WalkDirection {
+    FORWARD_ONLY, FORWARD_PREFERRED, FORWARD_UNIQUE, RANDOM
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/WalkMode.java

@@ -0,0 +1,25 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.enums;
+
+public enum WalkMode {
+    RANDOM, WEIGHTED, POPULARITY, NEAREST,
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/exception/NoEdgesException.java

@@ -0,0 +1,37 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.exception;
+
+public class NoEdgesException extends RuntimeException {
+
+    public NoEdgesException() {
+        super();
+    }
+
+    public NoEdgesException(String s) {
+        super(s);
+    }
+
+    public NoEdgesException(String s, Exception e) {
+        super(s, e);
+    }
+
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/exception/ParseException.java

@@ -0,0 +1,35 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.exception;
+
+public class ParseException extends RuntimeException {
+    public ParseException() {
+        super();
+    }
+
+    public ParseException(String s) {
+        super(s);
+    }
+
+    public ParseException(String s, Exception e) {
+        super(s, e);
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/huffman/BinaryTree.java

@@ -0,0 +1,32 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.huffman;
+
+public interface BinaryTree {
+
+    long getCode(int element);
+
+    int getCodeLength(int element);
+
+    String getCodeString(int element);
+
+    int[] getPathInnerNodes(int element);
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/huffman/GraphHuffman.java

@@ -0,0 +1,163 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.huffman;
+
+import lombok.AllArgsConstructor;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.PriorityQueue;
+
+public class GraphHuffman implements BinaryTree {
+    private final int MAX_CODE_LENGTH;
+    private final long[] codes;
+    private final byte[] codeLength;
+    private final int[][] innerNodePathToLeaf;
+
+    /**
+     * @param nVertices number of vertices in the graph that this Huffman tree is being built for
+     */
+    public GraphHuffman(int nVertices) {
+        this(nVertices, 64);
+    }
+
+    /**
+     * @param nVertices nVertices number of vertices in the graph that this Huffman tree is being built for
+     * @param maxCodeLength MAX_CODE_LENGTH for Huffman tree
+     */
+    public GraphHuffman(int nVertices, int maxCodeLength) {
+        this.codes = new long[nVertices];
+        this.codeLength = new byte[nVertices];
+        this.innerNodePathToLeaf = new int[nVertices][0];
+        this.MAX_CODE_LENGTH = maxCodeLength;
+    }
+
+    /** Build the Huffman tree given an array of vertex degrees
+     * @param vertexDegree vertexDegree[i] = degree of ith vertex
+     */
+    public void buildTree(int[] vertexDegree) {
+        PriorityQueue<Node> pq = new PriorityQueue<>();
+        for (int i = 0; i < vertexDegree.length; i++)
+            pq.add(new Node(i, vertexDegree[i], null, null));
+
+        while (pq.size() > 1) {
+            Node left = pq.remove();
+            Node right = pq.remove();
+            Node newNode = new Node(-1, left.count + right.count, left, right);
+            pq.add(newNode);
+        }
+
+        //Eventually: only one node left -> full tree
+        Node tree = pq.remove();
+
+        //Now: convert tree into binary codes. Traverse tree (preorder traversal) -> record path (left/right) -> code
+        int[] innerNodePath = new int[MAX_CODE_LENGTH];
+        traverse(tree, 0L, (byte) 0, -1, innerNodePath, 0);
+    }
+
+    @AllArgsConstructor
+    private static class Node implements Comparable<Node> {
+        private final int vertexIdx;
+        private final long count;
+        private Node left;
+        private Node right;
+
+        @Override
+        public int compareTo(Node o) {
+            return Long.compare(count, o.count);
+        }
+    }
+
+    private int traverse(Node node, long codeSoFar, byte codeLengthSoFar, int innerNodeCount, int[] innerNodePath,
+                    int currDepth) {
+        if (codeLengthSoFar >= MAX_CODE_LENGTH)
+            throw new RuntimeException("Cannot generate code: code length exceeds " + MAX_CODE_LENGTH + " bits");
+        if (node.left == null && node.right == null) {
+            //Leaf node
+            codes[node.vertexIdx] = codeSoFar;
+            codeLength[node.vertexIdx] = codeLengthSoFar;
+            innerNodePathToLeaf[node.vertexIdx] = Arrays.copyOf(innerNodePath, currDepth);
+            return innerNodeCount;
+        }
+
+        //This is an inner node. It's index is 'innerNodeCount'
+        innerNodeCount++;
+        innerNodePath[currDepth] = innerNodeCount;
+
+        long codeLeft = setBit(codeSoFar, codeLengthSoFar, false);
+        innerNodeCount = traverse(node.left, codeLeft, (byte) (codeLengthSoFar + 1), innerNodeCount, innerNodePath,
+                        currDepth + 1);
+
+        long codeRight = setBit(codeSoFar, codeLengthSoFar, true);
+        innerNodeCount = traverse(node.right, codeRight, (byte) (codeLengthSoFar + 1), innerNodeCount, innerNodePath,
+                        currDepth + 1);
+        return innerNodeCount;
+    }
+
+    private static long setBit(long in, int bitNum, boolean value) {
+        if (value)
+            return (in | 1L << bitNum); //Bit mask |: 00010000
+        else
+            return (in & ~(1 << bitNum)); //Bit mask &: 11101111
+    }
+
+    private static boolean getBit(long in, int bitNum) {
+        long mask = 1L << bitNum;
+        return (in & mask) != 0L;
+    }
+
+    @Override
+    public long getCode(int vertexNum) {
+        return codes[vertexNum];
+    }
+
+    @Override
+    public int getCodeLength(int vertexNum) {
+        return codeLength[vertexNum];
+    }
+
+    @Override
+    public String getCodeString(int vertexNum) {
+        long code = codes[vertexNum];
+        int len = codeLength[vertexNum];
+        StringBuilder sb = new StringBuilder();
+        for (int i = 0; i < len; i++)
+            sb.append(getBit(code, i) ? "1" : "0");
+
+        return sb.toString();
+    }
+
+    public List<Integer> getCodeList(int vertexNum) {
+        List<Integer> result = new ArrayList<>();
+        long code = codes[vertexNum];
+        int len = codeLength[vertexNum];
+        for (int i = 0; i < len; i++) {
+            result.add(getBit(code, i) ? 1 : 0);
+        }
+        return result;
+    }
+
+    @Override
+    public int[] getPathInnerNodes(int vertexNum) {
+        return innerNodePathToLeaf[vertexNum];
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/Edge.java

@@ -0,0 +1,83 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.primitives;
+
+import lombok.Data;
+
+@Data
+public class Edge<T extends Number> {
+
+    private final int from;
+    private final int to;
+    private final T value;
+    private final boolean directed;
+
+    public Edge(int from, int to, T value, boolean directed) {
+        this.from = from;
+        this.to = to;
+        this.value = value;
+        this.directed = directed;
+    }
+
+    @Override
+    public String toString() {
+        return "edge(" + (directed ? "directed" : "undirected") + "," + from + (directed ? "->" : "--") + to + ","
+                        + (value != null ? value : "") + ")";
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (!(o instanceof Edge))
+            return false;
+        Edge<?> e = (Edge<?>) o;
+        if (directed != e.directed)
+            return false;
+        if (directed) {
+            if (from != e.from)
+                return false;
+            if (to != e.to)
+                return false;
+        } else {
+            if (from == e.from) {
+                if (to != e.to)
+                    return false;
+            } else {
+                if (from != e.to)
+                    return false;
+                if (to != e.from)
+                    return false;
+            }
+        }
+        if ((value != null && e.value == null) || (value == null && e.value != null))
+            return false;
+        return value == null || value.equals(e.value);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 17;
+        result = 31 * result + (directed ? 1 : 0);
+        result = 31 * result + from;
+        result = 31 * result + to;
+        result = 31 * result + (value == null ? 0 : value.hashCode());
+        return result;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/Graph.java

@@ -0,0 +1,296 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.primitives;
+
+import org.deeplearning4j.models.sequencevectors.graph.exception.NoEdgesException;
+import org.deeplearning4j.models.sequencevectors.graph.vertex.VertexFactory;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+
+import java.lang.reflect.Array;
+import java.util.*;
+
+public class Graph<V extends SequenceElement, E extends Number> implements IGraph<V, E> {
+    private boolean allowMultipleEdges;
+    private List<Edge<E>>[] edges; //edge[i].get(j).to = k, then edge from i -> k
+    private List<Vertex<V>> vertices;
+
+
+    public Graph(int numVertices, VertexFactory<V> vertexFactory) {
+        this(numVertices, false, vertexFactory);
+    }
+
+    @SuppressWarnings("unchecked")
+    public Graph(int numVertices, boolean allowMultipleEdges, VertexFactory<V> vertexFactory) {
+        if (numVertices <= 0)
+            throw new IllegalArgumentException();
+        this.allowMultipleEdges = allowMultipleEdges;
+
+        vertices = new ArrayList<>(numVertices);
+        for (int i = 0; i < numVertices; i++)
+            vertices.add(vertexFactory.create(i));
+
+        edges = (List<Edge<E>>[]) Array.newInstance(List.class, numVertices);
+    }
+
+
+    @SuppressWarnings("unchecked")
+    public Graph(List<Vertex<V>> vertices, boolean allowMultipleEdges) {
+        this.vertices = new ArrayList<>(vertices);
+        this.allowMultipleEdges = allowMultipleEdges;
+        edges = (List<Edge<E>>[]) Array.newInstance(List.class, vertices.size());
+    }
+
+    @SuppressWarnings("unchecked")
+    public Graph(Collection<V> elements, boolean allowMultipleEdges) {
+        this.vertices = new ArrayList<>();
+        this.allowMultipleEdges = allowMultipleEdges;
+        int idx = 0;
+        for (V element : elements) {
+            Vertex<V> vertex = new Vertex<>(idx, element);
+            vertices.add(vertex);
+            idx++;
+        }
+        edges = (List<Edge<E>>[]) Array.newInstance(List.class, vertices.size());
+    }
+
+    public Graph(List<Vertex<V>> vertices) {
+        this(vertices, false);
+    }
+
+    public Graph() {}
+
+    public void addVertex(Vertex<V> vertex, Edge<E> edge) {
+        this.addEdge(edge);
+    }
+
+    public void addVertex(Vertex<V> vertex, Collection<Edge<E>> edges) {
+
+        for (Edge<E> edge : edges) {
+            this.addEdge(edge);
+        }
+    }
+
+    @Override
+    public int numVertices() {
+        return vertices.size();
+    }
+
+    @Override
+    public Vertex<V> getVertex(int idx) {
+        if (idx < 0 || idx >= vertices.size())
+            throw new IllegalArgumentException("Invalid index: " + idx);
+        return vertices.get(idx);
+    }
+
+    @Override
+    public List<Vertex<V>> getVertices(int[] indexes) {
+        List<Vertex<V>> out = new ArrayList<>(indexes.length);
+        for (int i : indexes)
+            out.add(getVertex(i));
+        return out;
+    }
+
+    @Override
+    public List<Vertex<V>> getVertices(int from, int to) {
+        if (to < from || from < 0 || to >= vertices.size())
+            throw new IllegalArgumentException("Invalid range: from=" + from + ", to=" + to);
+        List<Vertex<V>> out = new ArrayList<>(to - from + 1);
+        for (int i = from; i <= to; i++)
+            out.add(getVertex(i));
+        return out;
+    }
+
+    @Override
+    public void addEdge(Edge<E> edge) {
+        if (edge.getFrom() < 0 || edge.getTo() >= vertices.size())
+            throw new IllegalArgumentException("Invalid edge: " + edge + ", from/to indexes out of range");
+
+        List<Edge<E>> fromList = edges[edge.getFrom()];
+        if (fromList == null) {
+            fromList = new ArrayList<>();
+            edges[edge.getFrom()] = fromList;
+        }
+        addEdgeHelper(edge, fromList);
+
+        if (edge.isDirected())
+            return;
+
+        //Add other way too (to allow easy lookup for undirected edges)
+        List<Edge<E>> toList = edges[edge.getTo()];
+        if (toList == null) {
+            toList = new ArrayList<>();
+            edges[edge.getTo()] = toList;
+        }
+        addEdgeHelper(edge, toList);
+    }
+
+    /**
+     * Convenience method for adding an edge (directed or undirected) to graph
+     *
+     * @param from
+     * @param to
+     * @param value
+     * @param directed
+     */
+    @Override
+    public void addEdge(int from, int to, E value, boolean directed) {
+        addEdge(new Edge<>(from, to, value, directed));
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public List<Edge<E>> getEdgesOut(int vertex) {
+        if (edges[vertex] == null)
+            return Collections.emptyList();
+        return new ArrayList<>(edges[vertex]);
+    }
+
+    @Override
+    public int getVertexDegree(int vertex) {
+        if (edges[vertex] == null)
+            return 0;
+        return edges[vertex].size();
+    }
+
+    @Override
+    public Vertex<V> getRandomConnectedVertex(int vertex, Random rng) throws NoEdgesException {
+        if (vertex < 0 || vertex >= vertices.size())
+            throw new IllegalArgumentException("Invalid vertex index: " + vertex);
+        if (edges[vertex] == null || edges[vertex].isEmpty())
+            throw new NoEdgesException("Cannot generate random connected vertex: vertex " + vertex
+                            + " has no outgoing/undirected edges");
+        int connectedVertexNum = rng.nextInt(edges[vertex].size());
+        Edge<E> edge = edges[vertex].get(connectedVertexNum);
+        if (edge.getFrom() == vertex)
+            return vertices.get(edge.getTo()); //directed or undirected, vertex -> x
+        else
+            return vertices.get(edge.getFrom()); //Undirected edge, x -> vertex
+    }
+
+    @Override
+    public List<Vertex<V>> getConnectedVertices(int vertex) {
+        if (vertex < 0 || vertex >= vertices.size())
+            throw new IllegalArgumentException("Invalid vertex index: " + vertex);
+
+        if (edges[vertex] == null)
+            return Collections.emptyList();
+        List<Vertex<V>> list = new ArrayList<>(edges[vertex].size());
+        for (Edge<E> edge : edges[vertex]) {
+            list.add(vertices.get(edge.getTo()));
+        }
+        return list;
+    }
+
+    @Override
+    public int[] getConnectedVertexIndices(int vertex) {
+        int[] out = new int[(edges[vertex] == null ? 0 : edges[vertex].size())];
+        if (out.length == 0)
+            return out;
+        for (int i = 0; i < out.length; i++) {
+            Edge<E> e = edges[vertex].get(i);
+            out[i] = (e.getFrom() == vertex ? e.getTo() : e.getFrom());
+        }
+        return out;
+    }
+
+    private void addEdgeHelper(Edge<E> edge, List<Edge<E>> list) {
+        if (!allowMultipleEdges) {
+            //Check to avoid multiple edges
+            boolean duplicate = false;
+
+            if (edge.isDirected()) {
+                for (Edge<E> e : list) {
+                    if (e.getTo() == edge.getTo()) {
+                        duplicate = true;
+                        break;
+                    }
+                }
+            } else {
+                for (Edge<E> e : list) {
+                    if ((e.getFrom() == edge.getFrom() && e.getTo() == edge.getTo())
+                                    || (e.getTo() == edge.getFrom() && e.getFrom() == edge.getTo())) {
+                        duplicate = true;
+                        break;
+                    }
+                }
+            }
+
+            if (!duplicate) {
+                list.add(edge);
+            }
+        } else {
+            //allow multiple/duplicate edges
+            list.add(edge);
+        }
+    }
+
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("Graph {");
+        sb.append("\nVertices {");
+        for (Vertex<V> v : vertices) {
+            sb.append("\n\t").append(v);
+        }
+        sb.append("\n}");
+        sb.append("\nEdges {");
+        for (int i = 0; i < edges.length; i++) {
+            sb.append("\n\t");
+            if (edges[i] == null)
+                continue;
+            sb.append(i).append(":");
+            for (Edge<E> e : edges[i]) {
+                sb.append(" ").append(e);
+            }
+        }
+        sb.append("\n}");
+        sb.append("\n}");
+        return sb.toString();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (!(o instanceof Graph))
+            return false;
+        Graph g = (Graph) o;
+        if (allowMultipleEdges != g.allowMultipleEdges)
+            return false;
+        if (edges.length != g.edges.length)
+            return false;
+        if (vertices.size() != g.vertices.size())
+            return false;
+        for (int i = 0; i < edges.length; i++) {
+            if (!edges[i].equals(g.edges[i]))
+                return false;
+        }
+        return vertices.equals(g.vertices);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 23;
+        result = 31 * result + (allowMultipleEdges ? 1 : 0);
+        result = 31 * result + Arrays.hashCode(edges);
+        result = 31 * result + vertices.hashCode();
+        return result;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/IGraph.java

@@ -0,0 +1,105 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.primitives;
+
+
+import org.deeplearning4j.models.sequencevectors.graph.exception.NoEdgesException;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+
+import java.util.List;
+import java.util.Random;
+
+public interface IGraph<V extends SequenceElement, E extends Number> {
+
+    /** Number of vertices in the graph */
+    public int numVertices();
+
+    /**Get a vertex in the graph for a given index
+     * @param idx integer index of the vertex to get. must be in range 0 to numVertices()
+     * @return vertex
+     */
+    public Vertex<V> getVertex(int idx);
+
+    /** Get multiple vertices in the graph
+     * @param indexes the indexes of the vertices to retrieve
+     * @return list of vertices
+     */
+    public List<Vertex<V>> getVertices(int[] indexes);
+
+    /** Get multiple vertices in the graph, with secified indices
+     * @param from first vertex to get, inclusive
+     * @param to last vertex to get, inclusive
+     * @return list of vertices
+     */
+    public List<Vertex<V>> getVertices(int from, int to);
+
+    /** Add an edge to the graph.
+     */
+    public void addEdge(Edge<E> edge);
+
+    /** Convenience method for adding an edge (directed or undirected) to graph */
+    public void addEdge(int from, int to, E value, boolean directed);
+
+    /** Returns a list of edges for a vertex with a given index
+     * For undirected graphs, returns all edges incident on the vertex
+     * For directed graphs, only returns outward directed edges
+     * @param vertex index of the vertex to
+     * @return list of edges for this vertex
+     */
+    public List<Edge<E>> getEdgesOut(int vertex);
+
+    /** Returns the degree of the vertex.<br>
+     * For undirected graphs, this is just the degree.<br>
+     * For directed graphs, this returns the outdegree
+     * @param vertex vertex to get degree for
+     * @return vertex degree
+     */
+    public int getVertexDegree(int vertex);
+
+    /** Randomly sample a vertex connected to a given vertex. Sampling is done uniformly at random.
+     * Specifically, returns a random X such that either a directed edge (vertex -> X) exists,
+     * or an undirected edge (vertex -- X) exists<br>
+     * Can be used for example to implement a random walk on the graph (specifically: a unweighted random walk)
+     * @param vertex vertex to randomly sample from
+     * @param rng Random number generator to use
+     * @return A vertex connected to the specified vertex,
+     * @throws NoEdgesException thrown if the specified vertex has no edges, or no outgoing edges (in the case
+     * of a directed graph).
+     */
+    public Vertex<V> getRandomConnectedVertex(int vertex, Random rng) throws NoEdgesException;
+
+    /**Get a list of all of the vertices that the specified vertex is connected to<br>
+     * Specifically, for undirected graphs return list of all X such that (vertex -- X) exists<br>
+     * For directed graphs, return list of all X such that (vertex -> X) exists
+     * @param vertex Index of the vertex
+     * @return list of vertices that the specified vertex is connected to
+     */
+    public List<Vertex<V>> getConnectedVertices(int vertex);
+
+    /**Return an array of indexes of vertices that the specified vertex is connected to.<br>
+     * Specifically, for undirected graphs return int[] of all X.vertexID() such that (vertex -- X) exists<br>
+     * For directed graphs, return int[] of all X.vertexID() such that (vertex -> X) exists
+     * @param vertex index of the vertex
+     * @return list of vertices that the specified vertex is connected to
+     * @see #getConnectedVertices(int)
+     */
+    public int[] getConnectedVertexIndices(int vertex);
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/Vertex.java

@@ -0,0 +1,65 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.primitives;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+
+@AllArgsConstructor
+public class Vertex<T extends SequenceElement> {
+
+    private final int idx;
+    @Getter
+    @Setter
+    private T value;
+
+    public int vertexID() {
+        return idx;
+    }
+
+
+    @Override
+    public String toString() {
+        return "vertex(" + idx + "," + (value != null ? value : "") + ")";
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (!(o instanceof Vertex))
+            return false;
+        Vertex<?> v = (Vertex<?>) o;
+        if (idx != v.idx)
+            return false;
+        if ((value == null && v.value != null) || (value != null && v.value == null))
+            return false;
+        return value == null || value.equals(v.value);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 17;
+        result = 31 * result + idx;
+        result = 31 * result + (value == null ? 0 : value.hashCode());
+        return result;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/vertex/AbstractVertexFactory.java

@@ -0,0 +1,39 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.vertex;
+
+import org.deeplearning4j.models.sequencevectors.graph.primitives.Vertex;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+
+public class AbstractVertexFactory<T extends SequenceElement> implements VertexFactory<T> {
+
+    @Override
+    public Vertex<T> create(int vertexIdx) {
+        Vertex<T> vertex = new Vertex<>(vertexIdx, null);
+        return vertex;
+    }
+
+    @Override
+    public Vertex<T> create(int vertexIdx, T element) {
+        Vertex<T> vertex = new Vertex<>(vertexIdx, element);
+        return vertex;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/vertex/VertexFactory.java

@@ -0,0 +1,32 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.vertex;
+
+
+import org.deeplearning4j.models.sequencevectors.graph.primitives.Vertex;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+
+public interface VertexFactory<T extends SequenceElement> {
+
+    Vertex<T> create(int vertexIdx);
+
+    Vertex<T> create(int vertexIdx, T element);
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/GraphWalker.java

@@ -0,0 +1,54 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.walkers;
+
+import org.deeplearning4j.models.sequencevectors.graph.primitives.IGraph;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+
+public interface GraphWalker<T extends SequenceElement> {
+
+    IGraph<T, ?> getSourceGraph();
+
+    /**
+     * This method checks, if walker has any more sequences left in queue
+     *
+     * @return
+     */
+    boolean hasNext();
+
+    /**
+     * This method returns next walk sequence from this graph
+     *
+     * @return
+     */
+    Sequence<T> next();
+
+    /**
+     * This method resets walker
+     *
+     * @param shuffle if TRUE, order of walks will be shuffled
+     */
+    void reset(boolean shuffle);
+
+
+    boolean isLabelEnabled();
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/NearestVertexWalker.java

@@ -0,0 +1,274 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.walkers.impl;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.models.sequencevectors.graph.enums.SamplingMode;
+import org.deeplearning4j.models.sequencevectors.graph.primitives.IGraph;
+import org.deeplearning4j.models.sequencevectors.graph.primitives.Vertex;
+import org.deeplearning4j.models.sequencevectors.graph.walkers.GraphWalker;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.nd4j.linalg.exception.ND4JIllegalStateException;
+import org.nd4j.common.util.ArrayUtil;
+
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+@Slf4j
+public class NearestVertexWalker<V extends SequenceElement> implements GraphWalker<V> {
+    @Getter
+    protected IGraph<V, ?> sourceGraph;
+    protected int walkLength = 0;
+    protected long seed = 0;
+    protected SamplingMode samplingMode = SamplingMode.RANDOM;
+    protected int[] order;
+    protected Random rng;
+    protected int depth;
+
+    private AtomicInteger position = new AtomicInteger(0);
+
+    protected NearestVertexWalker() {
+
+    }
+
+    @Override
+    public boolean hasNext() {
+        return position.get() < order.length;
+    }
+
+    @Override
+    public Sequence<V> next() {
+        return walk(sourceGraph.getVertex(order[position.getAndIncrement()]), 1);
+    }
+
+    @Override
+    public void reset(boolean shuffle) {
+        position.set(0);
+        if (shuffle) {
+            log.trace("Calling shuffle() on entries...");
+            // https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle#The_modern_algorithm
+            for (int i = order.length - 1; i > 0; i--) {
+                int j = rng.nextInt(i + 1);
+                int temp = order[j];
+                order[j] = order[i];
+                order[i] = temp;
+            }
+        }
+    }
+
+    protected Sequence<V> walk(Vertex<V> node, int cDepth) {
+        Sequence<V> sequence = new Sequence<>();
+
+        int idx = node.vertexID();
+        List<Vertex<V>> vertices = sourceGraph.getConnectedVertices(idx);
+
+        sequence.setSequenceLabel(node.getValue());
+
+        if (walkLength == 0) {
+            // if walk is unlimited - we use all connected vertices as is
+            for (Vertex<V> vertex : vertices)
+                sequence.addElement(vertex.getValue());
+        } else {
+            // if walks are limited, we care about sampling mode
+            switch (samplingMode) {
+                case MAX_POPULARITY: {
+                    Collections.sort(vertices, new VertexComparator<>(sourceGraph));
+                    for (int i = 0; i < walkLength; i++) {
+                        sequence.addElement(vertices.get(i).getValue());
+
+                        // going for one more depth level
+                        if (depth > 1 && cDepth < depth) {
+                            Sequence<V> nextDepth = walk(vertices.get(i), ++cDepth);
+                            for (V element : nextDepth.getElements()) {
+                                if (sequence.getElementByLabel(element.getLabel()) == null)
+                                    sequence.addElement(element);
+                            }
+                        }
+                    }
+
+                }
+                case MEDIAN_POPULARITY: {
+                    Collections.sort(vertices, new VertexComparator<>(sourceGraph));
+                    for (int i = (vertices.size() / 2) - (walkLength / 2), e = 0; e < walkLength
+                                    && i < vertices.size(); i++, e++) {
+                        sequence.addElement(vertices.get(i).getValue());
+
+                        // going for one more depth level
+                        if (depth > 1 && cDepth < depth) {
+                            Sequence<V> nextDepth = walk(vertices.get(i), ++cDepth);
+                            for (V element : nextDepth.getElements()) {
+                                if (sequence.getElementByLabel(element.getLabel()) == null)
+                                    sequence.addElement(element);
+                            }
+                        }
+                    }
+
+                }
+                case MIN_POPULARITY: {
+                    Collections.sort(vertices, new VertexComparator<>(sourceGraph));
+                    for (int i = vertices.size(), e = 0; e < walkLength && i >= 0; i--, e++) {
+                        sequence.addElement(vertices.get(i).getValue());
+
+                        // going for one more depth level
+                        if (depth > 1 && cDepth < depth) {
+                            Sequence<V> nextDepth = walk(vertices.get(i), ++cDepth);
+                            for (V element : nextDepth.getElements()) {
+                                if (sequence.getElementByLabel(element.getLabel()) == null)
+                                    sequence.addElement(element);
+                            }
+                        }
+                    }
+                }
+                case RANDOM: {
+                    // we randomly sample some number of connected vertices
+                    if (vertices.size() <= walkLength)
+                        for (Vertex<V> vertex : vertices)
+                            sequence.addElement(vertex.getValue());
+                    else {
+                        Set<V> elements = new HashSet<>();
+                        while (elements.size() < walkLength) {
+                            Vertex<V> vertex = ArrayUtil.getRandomElement(vertices);
+                            elements.add(vertex.getValue());
+
+                            // going for one more depth level
+                            if (depth > 1 && cDepth < depth) {
+                                Sequence<V> nextDepth = walk(vertex, ++cDepth);
+                                for (V element : nextDepth.getElements()) {
+                                    if (sequence.getElementByLabel(element.getLabel()) == null)
+                                        sequence.addElement(element);
+                                }
+                            }
+                        }
+
+                        sequence.addElements(elements);
+                    }
+                }
+                    break;
+                default:
+                    throw new ND4JIllegalStateException("Unknown sampling mode was passed in: [" + samplingMode + "]");
+            }
+        }
+
+        return sequence;
+    }
+
+    @Override
+    public boolean isLabelEnabled() {
+        return true;
+    }
+
+    public static class Builder<V extends SequenceElement> {
+        protected int walkLength = 0;
+        protected IGraph<V, ?> sourceGraph;
+        protected SamplingMode samplingMode = SamplingMode.RANDOM;
+        protected long seed;
+        protected int depth = 1;
+
+        public Builder(@NonNull IGraph<V, ?> graph) {
+            this.sourceGraph = graph;
+        }
+
+        public Builder setSeed(long seed) {
+            this.seed = seed;
+            return this;
+        }
+
+        /**
+         * This method defines maximal number of nodes to be visited during walk.
+         *
+         * PLEASE NOTE: If set to 0 - no limits will be used.
+         *
+         * Default value: 0
+         * @param length
+         * @return
+         */
+        public Builder setWalkLength(int length) {
+            walkLength = length;
+            return this;
+        }
+
+        /**
+         * This method specifies, how deep walker goes from starting point
+         *
+         * Default value: 1
+         * @param depth
+         * @return
+         */
+        public Builder setDepth(int depth) {
+            this.depth = depth;
+            return this;
+        }
+
+        /**
+         * This method defines sorting which will be used to generate walks.
+         *
+         * PLEASE NOTE: This option has effect only if walkLength is limited (>0).
+         *
+         * @param mode
+         * @return
+         */
+        public Builder setSamplingMode(@NonNull SamplingMode mode) {
+            this.samplingMode = mode;
+            return this;
+        }
+
+        /**
+         * This method returns you new GraphWalker instance
+         *
+         * @return
+         */
+        public NearestVertexWalker<V> build() {
+            NearestVertexWalker<V> walker = new NearestVertexWalker<>();
+            walker.sourceGraph = this.sourceGraph;
+            walker.walkLength = this.walkLength;
+            walker.samplingMode = this.samplingMode;
+            walker.depth = this.depth;
+
+            walker.order = new int[sourceGraph.numVertices()];
+            for (int i = 0; i < walker.order.length; i++) {
+                walker.order[i] = i;
+            }
+
+            walker.rng = new Random(seed);
+
+            walker.reset(true);
+
+            return walker;
+        }
+    }
+
+    protected class VertexComparator<V extends SequenceElement, E extends Number> implements Comparator<Vertex<V>> {
+        private IGraph<V, E> graph;
+
+        public VertexComparator(@NonNull IGraph<V, E> graph) {
+            this.graph = graph;
+        }
+
+        @Override
+        public int compare(Vertex<V> o1, Vertex<V> o2) {
+            return Integer.compare(graph.getConnectedVertices(o2.vertexID()).size(),
+                            graph.getConnectedVertices(o1.vertexID()).size());
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/PopularityWalker.java

@@ -0,0 +1,384 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.walkers.impl;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NonNull;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.commons.math3.util.MathArrays;
+import org.deeplearning4j.models.sequencevectors.graph.enums.NoEdgeHandling;
+import org.deeplearning4j.models.sequencevectors.graph.enums.PopularityMode;
+import org.deeplearning4j.models.sequencevectors.graph.enums.SpreadSpectrum;
+import org.deeplearning4j.models.sequencevectors.graph.enums.WalkDirection;
+import org.deeplearning4j.models.sequencevectors.graph.exception.NoEdgesException;
+import org.deeplearning4j.models.sequencevectors.graph.primitives.IGraph;
+import org.deeplearning4j.models.sequencevectors.graph.primitives.Vertex;
+import org.deeplearning4j.models.sequencevectors.graph.walkers.GraphWalker;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+public class PopularityWalker<T extends SequenceElement> extends RandomWalker<T> implements GraphWalker<T> {
+    protected PopularityMode popularityMode = PopularityMode.MAXIMUM;
+    protected int spread = 10;
+    protected SpreadSpectrum spectrum;
+
+    private static final Logger logger = LoggerFactory.getLogger(PopularityWalker.class);
+
+    /**
+     * This method checks, if walker has any more sequences left in queue
+     *
+     * @return
+     */
+    @Override
+    public boolean hasNext() {
+        return super.hasNext();
+    }
+
+    @Override
+    public boolean isLabelEnabled() {
+        return false;
+    }
+
+
+    protected class NodeComparator implements Comparator<Node<T>> {
+
+        @Override
+        public int compare(Node<T> o1, Node<T> o2) {
+            return Integer.compare(o2.weight, o1.weight);
+        }
+    }
+
+    /**
+     * This method returns next walk sequence from this graph
+     *
+     * @return
+     */
+    @Override
+    public Sequence<T> next() {
+        Sequence<T> sequence = new Sequence<>();
+        int[] visitedHops = new int[walkLength];
+        Arrays.fill(visitedHops, -1);
+
+        int startPosition = position.getAndIncrement();
+        int lastId = -1;
+        int startPoint = order[startPosition];
+        startPosition = startPoint;
+        for (int i = 0; i < walkLength; i++) {
+
+            Vertex<T> vertex = sourceGraph.getVertex(startPosition);
+
+            int currentPosition = startPosition;
+
+            sequence.addElement(vertex.getValue());
+            visitedHops[i] = vertex.vertexID();
+            int cSpread = 0;
+
+            if (alpha > 0 && lastId != startPoint && lastId != -1 && alpha > rng.nextDouble()) {
+                startPosition = startPoint;
+                continue;
+            }
+
+            switch (walkDirection) {
+                case RANDOM:
+                case FORWARD_ONLY:
+                case FORWARD_UNIQUE:
+                case FORWARD_PREFERRED: {
+
+                    // ArrayUtils.removeElements(sourceGraph.getConnectedVertexIndices(order[currentPosition]), visitedHops);
+                    int[] connections = ArrayUtils.removeElements(
+                                    sourceGraph.getConnectedVertexIndices(vertex.vertexID()), visitedHops);
+
+                    // we get  popularity of each node connected to the current node.
+                    PriorityQueue<Node<T>> queue = new PriorityQueue<>(Math.max(10, connections.length), new NodeComparator());
+
+                    int start = 0;
+                    int stop = 0;
+                    int cnt = 0;
+                    if (connections.length > 0) {
+
+
+                        for (int connected : connections) {
+                            Node<T> tNode = new Node<>(connected, sourceGraph.getConnectedVertices(connected).size());
+                            queue.add(tNode);
+                        }
+
+
+                        cSpread = spread > connections.length ? connections.length : spread;
+
+                        switch (popularityMode) {
+                            case MAXIMUM:
+                                start = 0;
+                                stop = start + cSpread - 1;
+                                break;
+                            case MINIMUM:
+                                start = connections.length - cSpread;
+                                stop = connections.length - 1;
+                                break;
+                            case AVERAGE:
+                                int mid = connections.length / 2;
+                                start = mid - (cSpread / 2);
+                                stop = mid + (cSpread / 2);
+                                break;
+                        }
+
+                        // logger.info("Spread: ["+ cSpread+ "], Connections: ["+ connections.length+"], Start: ["+start+"], Stop: ["+stop+"]");
+                        cnt = 0;
+                        //logger.info("Queue: " + queue);
+                        //logger.info("Queue size: " + queue.size());
+
+                        List<Node<T>> list = new ArrayList<>();
+                        double[] weights = new double[cSpread];
+
+                        int fcnt = 0;
+                        while (!queue.isEmpty()) {
+                            Node<T> node = queue.poll();
+                            if (cnt >= start && cnt <= stop) {
+                                list.add(node);
+                                weights[fcnt] = node.getWeight();
+                                fcnt++;
+                            }
+                            connections[cnt] = node.getVertexId();
+
+                            cnt++;
+                        }
+
+
+                        int con = -1;
+
+                        switch (spectrum) {
+                            case PLAIN: {
+                                con = RandomUtils.nextInt(start, stop + 1);
+
+                                //    logger.info("Picked selection: " + con);
+
+                                Vertex<T> nV = sourceGraph.getVertex(connections[con]);
+                                startPosition = nV.vertexID();
+                                lastId = vertex.vertexID();
+                            }
+                                break;
+                            case PROPORTIONAL: {
+                                double norm[] = MathArrays.normalizeArray(weights, 1);
+                                double prob = rng.nextDouble();
+                                double floor = 0.0;
+                                for (int b = 0; b < weights.length; b++) {
+                                    if (prob >= floor && prob < floor + norm[b]) {
+                                        startPosition = list.get(b).getVertexId();
+                                        lastId = startPosition;
+                                        break;
+                                    } else {
+                                        floor += norm[b];
+                                    }
+                                }
+                            }
+                                break;
+                        }
+
+                    } else {
+                        switch (noEdgeHandling) {
+                            case EXCEPTION_ON_DISCONNECTED:
+                                throw new NoEdgesException("No more edges at vertex [" + currentPosition + "]");
+                            case CUTOFF_ON_DISCONNECTED:
+                                i += walkLength;
+                                break;
+                            case SELF_LOOP_ON_DISCONNECTED:
+                                startPosition = currentPosition;
+                                break;
+                            case RESTART_ON_DISCONNECTED:
+                                startPosition = startPoint;
+                                break;
+                            default:
+                                throw new UnsupportedOperationException(
+                                                "Unsupported noEdgeHandling: [" + noEdgeHandling + "]");
+                        }
+                    }
+                }
+                    break;
+                default:
+                    throw new UnsupportedOperationException("Unknown WalkDirection: [" + walkDirection + "]");
+            }
+
+        }
+
+        return sequence;
+    }
+
+    @Override
+    public void reset(boolean shuffle) {
+        super.reset(shuffle);
+    }
+
+    public static class Builder<T extends SequenceElement> extends RandomWalker.Builder<T> {
+        protected PopularityMode popularityMode = PopularityMode.MAXIMUM;
+        protected int spread = 10;
+        protected SpreadSpectrum spectrum = SpreadSpectrum.PLAIN;
+
+        public Builder(IGraph<T, ?> sourceGraph) {
+            super(sourceGraph);
+        }
+
+
+        /**
+         * This method defines which nodes should be taken in account when choosing next hope: maximum popularity, lowest popularity, or average popularity.
+         * Default value: MAXIMUM
+         *
+         * @param popularityMode
+         * @return
+         */
+        public Builder<T> setPopularityMode(@NonNull PopularityMode popularityMode) {
+            this.popularityMode = popularityMode;
+            return this;
+        }
+
+        /**
+         * This method defines, how much nodes should take place in next hop selection. Something like top-N nodes, or bottom-N nodes.
+         * Default value: 10
+         *
+         * @param topN
+         * @return
+         */
+        public Builder<T> setPopularitySpread(int topN) {
+            this.spread = topN;
+            return this;
+        }
+
+        /**
+         * This method allows you to define, if nodes within popularity spread should have equal chances to be picked for next hop, or they should have chances proportional to their popularity.
+         *
+         * Default value: PLAIN
+         *
+         * @param spectrum
+         * @return
+         */
+        public Builder<T> setSpreadSpectrum(@NonNull SpreadSpectrum spectrum) {
+            this.spectrum = spectrum;
+            return this;
+        }
+
+        /**
+         * This method defines walker behavior when it gets to node which has no next nodes available
+         * Default value: RESTART_ON_DISCONNECTED
+         *
+         * @param handling
+         * @return
+         */
+        @Override
+        public Builder<T> setNoEdgeHandling(@NonNull NoEdgeHandling handling) {
+            super.setNoEdgeHandling(handling);
+            return this;
+        }
+
+        /**
+         * This method specifies random seed.
+         *
+         * @param seed
+         * @return
+         */
+        @Override
+        public Builder<T> setSeed(long seed) {
+            super.setSeed(seed);
+            return this;
+        }
+
+        /**
+         * This method defines next hop selection within walk
+         *
+         * @param direction
+         * @return
+         */
+        @Override
+        public Builder<T> setWalkDirection(@NonNull WalkDirection direction) {
+            super.setWalkDirection(direction);
+            return this;
+        }
+
+        /**
+         * This method specifies output sequence (walk) length
+         *
+         * @param walkLength
+         * @return
+         */
+        @Override
+        public Builder<T> setWalkLength(int walkLength) {
+            super.setWalkLength(walkLength);
+            return this;
+        }
+
+        /**
+         * This method defines a chance for walk restart
+         * Good value would be somewhere between 0.03-0.07
+         *
+         * @param alpha
+         * @return
+         */
+        @Override
+        public Builder<T> setRestartProbability(double alpha) {
+            super.setRestartProbability(alpha);
+            return this;
+        }
+
+        /**
+         * This method builds PopularityWalker object with previously specified params
+         *
+         * @return
+         */
+        @Override
+        public PopularityWalker<T> build() {
+            PopularityWalker<T> walker = new PopularityWalker<>();
+            walker.noEdgeHandling = this.noEdgeHandling;
+            walker.sourceGraph = this.sourceGraph;
+            walker.walkLength = this.walkLength;
+            walker.seed = this.seed;
+            walker.walkDirection = this.walkDirection;
+            walker.alpha = this.alpha;
+            walker.popularityMode = this.popularityMode;
+            walker.spread = this.spread;
+            walker.spectrum = this.spectrum;
+
+            walker.order = new int[sourceGraph.numVertices()];
+            for (int i = 0; i < walker.order.length; i++) {
+                walker.order[i] = i;
+            }
+
+            if (this.seed != 0)
+                walker.rng = new Random(this.seed);
+
+            return walker;
+        }
+    }
+
+    @AllArgsConstructor
+    @Data
+    private static class Node<T extends SequenceElement> implements Comparable<Node<T>> {
+        private int vertexId;
+        private int weight = 0;
+
+        @Override
+        public int compareTo(Node<T> o) {
+            return Integer.compare(this.weight, o.weight);
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/RandomWalker.java

@@ -0,0 +1,355 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.walkers.impl;
+
+import lombok.Getter;
+import lombok.NonNull;
+import org.apache.commons.lang3.ArrayUtils;
+import org.deeplearning4j.models.sequencevectors.graph.enums.NoEdgeHandling;
+import org.deeplearning4j.models.sequencevectors.graph.enums.WalkDirection;
+import org.deeplearning4j.models.sequencevectors.graph.exception.NoEdgesException;
+import org.deeplearning4j.models.sequencevectors.graph.primitives.IGraph;
+import org.deeplearning4j.models.sequencevectors.graph.primitives.Vertex;
+import org.deeplearning4j.models.sequencevectors.graph.walkers.GraphWalker;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class RandomWalker<T extends SequenceElement> implements GraphWalker<T> {
+    protected int walkLength = 5;
+    protected NoEdgeHandling noEdgeHandling = NoEdgeHandling.EXCEPTION_ON_DISCONNECTED;
+    @Getter
+    protected IGraph<T, ?> sourceGraph;
+    protected AtomicInteger position = new AtomicInteger(0);
+    protected Random rng = new Random(System.currentTimeMillis());
+    protected long seed;
+    protected int[] order;
+    protected WalkDirection walkDirection;
+    protected double alpha;
+
+    private static final Logger logger = LoggerFactory.getLogger(RandomWalker.class);
+
+    protected RandomWalker() {
+
+    }
+
+
+    /**
+     * This method checks, if walker has any more sequences left in queue
+     *
+     * @return
+     */
+    @Override
+    public boolean hasNext() {
+        return position.get() < sourceGraph.numVertices();
+    }
+
+    @Override
+    public boolean isLabelEnabled() {
+        return false;
+    }
+
+    /**
+     * This method returns next walk sequence from this graph
+     *
+     * @return
+     */
+    @Override
+    public Sequence<T> next() {
+        int[] visitedHops = new int[walkLength];
+        Arrays.fill(visitedHops, -1);
+
+        Sequence<T> sequence = new Sequence<>();
+
+        int startPosition = position.getAndIncrement();
+        int lastId = -1;
+        int startPoint = order[startPosition];
+        //System.out.println("");
+
+
+        startPosition = startPoint;
+
+        //if (startPosition == 0 || startPoint % 1000 == 0)
+        //   System.out.println("ATZ Walk: ");
+
+        for (int i = 0; i < walkLength; i++) {
+            Vertex<T> vertex = sourceGraph.getVertex(startPosition);
+
+            int currentPosition = startPosition;
+
+            sequence.addElement(vertex.getValue());
+            visitedHops[i] = vertex.vertexID();
+            //if (startPoint == 0 || startPoint % 1000 == 0)
+            // System.out.print("" + vertex.vertexID() + " -> ");
+
+
+            if (alpha > 0 && lastId != startPoint && lastId != -1 && alpha > rng.nextDouble()) {
+                startPosition = startPoint;
+                continue;
+            }
+
+
+            // get next vertex
+            switch (walkDirection) {
+                case RANDOM: {
+                    int[] nextHops = sourceGraph.getConnectedVertexIndices(currentPosition);
+                    startPosition = nextHops[rng.nextInt(nextHops.length)];
+                }
+                    break;
+                case FORWARD_ONLY: {
+                    // here we remove only last hop
+                    int[] nextHops = ArrayUtils.removeElements(sourceGraph.getConnectedVertexIndices(currentPosition),
+                                    lastId);
+                    if (nextHops.length > 0) {
+                        startPosition = nextHops[rng.nextInt(nextHops.length)];
+                    } else {
+                        switch (noEdgeHandling) {
+                            case CUTOFF_ON_DISCONNECTED: {
+                                i += walkLength;
+                            }
+                                break;
+                            case EXCEPTION_ON_DISCONNECTED: {
+                                throw new NoEdgesException("No more edges at vertex [" + currentPosition + "]");
+                            }
+                            case SELF_LOOP_ON_DISCONNECTED: {
+                                startPosition = currentPosition;
+                            }
+                                break;
+                            case PADDING_ON_DISCONNECTED: {
+                                throw new UnsupportedOperationException("PADDING not implemented yet");
+                            }
+                            case RESTART_ON_DISCONNECTED: {
+                                startPosition = startPoint;
+                            }
+                                break;
+                            default:
+                                throw new UnsupportedOperationException(
+                                                "NoEdgeHandling mode [" + noEdgeHandling + "] not implemented yet.");
+                        }
+                    }
+                }
+                    break;
+                case FORWARD_UNIQUE: {
+                    // here we remove all previously visited hops, and we don't get  back to them ever
+                    int[] nextHops = ArrayUtils.removeElements(sourceGraph.getConnectedVertexIndices(currentPosition),
+                                    visitedHops);
+                    if (nextHops.length > 0) {
+                        startPosition = nextHops[rng.nextInt(nextHops.length)];
+                    } else {
+                        // if we don't have any more unique hops within this path - break out.
+                        switch (noEdgeHandling) {
+                            case CUTOFF_ON_DISCONNECTED: {
+                                i += walkLength;
+                            }
+                                break;
+                            case EXCEPTION_ON_DISCONNECTED: {
+                                throw new NoEdgesException("No more edges at vertex [" + currentPosition + "]");
+                            }
+                            case SELF_LOOP_ON_DISCONNECTED: {
+                                startPosition = currentPosition;
+                            }
+                                break;
+                            case PADDING_ON_DISCONNECTED: {
+                                throw new UnsupportedOperationException("PADDING not implemented yet");
+                            }
+                            case RESTART_ON_DISCONNECTED: {
+                                startPosition = startPoint;
+                            }
+                                break;
+                            default:
+                                throw new UnsupportedOperationException(
+                                                "NoEdgeHandling mode [" + noEdgeHandling + "] not implemented yet.");
+                        }
+                    }
+                }
+                    break;
+                case FORWARD_PREFERRED: {
+                    // here we remove all previously visited hops, and if there's no next unique hop available - we fallback to anything, but the last one
+                    int[] nextHops = ArrayUtils.removeElements(sourceGraph.getConnectedVertexIndices(currentPosition),
+                                    visitedHops);
+                    if (nextHops.length == 0) {
+                        nextHops = ArrayUtils.removeElements(sourceGraph.getConnectedVertexIndices(currentPosition),
+                                        lastId);
+                        if (nextHops.length == 0) {
+                            switch (noEdgeHandling) {
+                                case CUTOFF_ON_DISCONNECTED: {
+                                    i += walkLength;
+                                }
+                                    break;
+                                case EXCEPTION_ON_DISCONNECTED: {
+                                    throw new NoEdgesException("No more edges at vertex [" + currentPosition + "]");
+                                }
+                                case SELF_LOOP_ON_DISCONNECTED: {
+                                    startPosition = currentPosition;
+                                }
+                                    break;
+                                case PADDING_ON_DISCONNECTED: {
+                                    throw new UnsupportedOperationException("PADDING not implemented yet");
+                                }
+                                case RESTART_ON_DISCONNECTED: {
+                                    startPosition = startPoint;
+                                }
+                                    break;
+                                default:
+                                    throw new UnsupportedOperationException("NoEdgeHandling mode [" + noEdgeHandling
+                                                    + "] not implemented yet.");
+                            }
+                        } else
+                            startPosition = nextHops[rng.nextInt(nextHops.length)];
+                    }
+                }
+                    break;
+                default:
+                    throw new UnsupportedOperationException("Unknown WalkDirection [" + walkDirection + "]");
+            }
+
+            lastId = vertex.vertexID();
+        }
+
+        //if (startPoint == 0 || startPoint % 1000 == 0)
+        //System.out.println("");
+        return sequence;
+    }
+
+    /**
+     * This method resets walker
+     *
+     * @param shuffle if TRUE, order of walks will be shuffled
+     */
+    @Override
+    public void reset(boolean shuffle) {
+        this.position.set(0);
+        if (shuffle) {
+            logger.trace("Calling shuffle() on entries...");
+            // https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle#The_modern_algorithm
+            for (int i = order.length - 1; i > 0; i--) {
+                int j = rng.nextInt(i + 1);
+                int temp = order[j];
+                order[j] = order[i];
+                order[i] = temp;
+            }
+        }
+    }
+
+    public static class Builder<T extends SequenceElement> {
+        protected int walkLength = 5;
+        protected NoEdgeHandling noEdgeHandling = NoEdgeHandling.RESTART_ON_DISCONNECTED;
+        protected IGraph<T, ?> sourceGraph;
+        protected long seed = 0;
+        protected WalkDirection walkDirection = WalkDirection.FORWARD_ONLY;
+        protected double alpha;
+
+        /**
+         * Builder constructor for RandomWalker
+         *
+         * @param graph source graph to be used for this walker
+         */
+        public Builder(@NonNull IGraph<T, ?> graph) {
+            this.sourceGraph = graph;
+        }
+
+        /**
+         * This method specifies output sequence (walk) length
+         *
+         * @param walkLength
+         * @return
+         */
+        public Builder<T> setWalkLength(int walkLength) {
+            this.walkLength = walkLength;
+            return this;
+        }
+
+        /**
+         * This method defines walker behavior when it gets to node which has no next nodes available
+         * Default value: RESTART_ON_DISCONNECTED
+         *
+         * @param handling
+         * @return
+         */
+        public Builder<T> setNoEdgeHandling(@NonNull NoEdgeHandling handling) {
+            this.noEdgeHandling = handling;
+            return this;
+        }
+
+        /**
+         * This method specifies random seed.
+         *
+         * @param seed
+         * @return
+         */
+        public Builder<T> setSeed(long seed) {
+            this.seed = seed;
+            return this;
+        }
+
+        /**
+         * This method defines next hop selection within walk
+         *
+         * @param direction
+         * @return
+         */
+        public Builder<T> setWalkDirection(@NonNull WalkDirection direction) {
+            this.walkDirection = direction;
+            return this;
+        }
+
+        /**
+         * This method defines a chance for walk restart
+         * Good value would be somewhere between 0.03-0.07
+         *
+         * @param alpha
+         * @return
+         */
+        public Builder<T> setRestartProbability(double alpha) {
+            this.alpha = alpha;
+            return this;
+        }
+
+        /**
+         * This method builds RandomWalker instance
+         * @return
+         */
+        public RandomWalker<T> build() {
+            RandomWalker<T> walker = new RandomWalker<>();
+            walker.noEdgeHandling = this.noEdgeHandling;
+            walker.sourceGraph = this.sourceGraph;
+            walker.walkLength = this.walkLength;
+            walker.seed = this.seed;
+            walker.walkDirection = this.walkDirection;
+            walker.alpha = this.alpha;
+
+            walker.order = new int[sourceGraph.numVertices()];
+            for (int i = 0; i < walker.order.length; i++) {
+                walker.order[i] = i;
+            }
+
+            if (this.seed != 0)
+                walker.rng = new Random(this.seed);
+
+            return walker;
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/WeightedWalker.java

@@ -0,0 +1,230 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.graph.walkers.impl;
+
+import lombok.NonNull;
+import org.deeplearning4j.models.sequencevectors.graph.enums.NoEdgeHandling;
+import org.deeplearning4j.models.sequencevectors.graph.enums.WalkDirection;
+import org.deeplearning4j.models.sequencevectors.graph.exception.NoEdgesException;
+import org.deeplearning4j.models.sequencevectors.graph.primitives.Edge;
+import org.deeplearning4j.models.sequencevectors.graph.primitives.IGraph;
+import org.deeplearning4j.models.sequencevectors.graph.primitives.Vertex;
+import org.deeplearning4j.models.sequencevectors.graph.walkers.GraphWalker;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+
+import java.util.List;
+import java.util.Random;
+
+public class WeightedWalker<T extends SequenceElement> extends RandomWalker<T> implements GraphWalker<T> {
+
+    protected WeightedWalker() {
+
+    }
+
+    /**
+     * This method checks, if walker has any more sequences left in queue
+     *
+     * @return
+     */
+    @Override
+    public boolean hasNext() {
+        return super.hasNext();
+    }
+
+    @Override
+    public boolean isLabelEnabled() {
+        return false;
+    }
+
+    /**
+     * This method returns next walk sequence from this graph
+     *
+     * @return
+     */
+    @Override
+    public Sequence<T> next() {
+        Sequence<T> sequence = new Sequence<>();
+
+        int startPosition = position.getAndIncrement();
+        int lastId = -1;
+        int currentPoint = order[startPosition];
+        int startPoint = currentPoint;
+        for (int i = 0; i < walkLength; i++) {
+
+            if (alpha > 0 && lastId != startPoint && lastId != -1 && alpha > rng.nextDouble()) {
+                startPosition = startPoint;
+                continue;
+            }
+
+
+            Vertex<T> vertex = sourceGraph.getVertex(currentPoint);
+            sequence.addElement(vertex.getValue());
+
+            List<? extends Edge<? extends Number>> edges = sourceGraph.getEdgesOut(currentPoint);
+
+            if (edges == null || edges.isEmpty()) {
+                switch (noEdgeHandling) {
+                    case CUTOFF_ON_DISCONNECTED:
+                        // we just break this sequence
+                        i = walkLength;
+                        break;
+                    case EXCEPTION_ON_DISCONNECTED:
+                        throw new NoEdgesException("No available edges left");
+                    case PADDING_ON_DISCONNECTED:
+                        // TODO: implement padding
+                        throw new UnsupportedOperationException("Padding isn't implemented yet");
+                    case RESTART_ON_DISCONNECTED:
+                        currentPoint = order[startPosition];
+                        break;
+                    case SELF_LOOP_ON_DISCONNECTED:
+                        // we pad walk with this vertex, to do that - we just don't do anything, and currentPoint will be the same till the end of walk
+                        break;
+                }
+            } else {
+                double totalWeight = 0.0;
+                for (Edge<? extends Number> edge : edges) {
+                    totalWeight += edge.getValue().doubleValue();
+                }
+
+                double d = rng.nextDouble();
+                double threshold = d * totalWeight;
+                double sumWeight = 0.0;
+                for (Edge<? extends Number> edge : edges) {
+                    sumWeight += edge.getValue().doubleValue();
+                    if (sumWeight >= threshold) {
+                        if (edge.isDirected()) {
+                            currentPoint = edge.getTo();
+                        } else {
+                            if (edge.getFrom() == currentPoint) {
+                                currentPoint = edge.getTo();
+                            } else {
+                                currentPoint = edge.getFrom(); //Undirected edge: might be next--currVertexIdx instead of currVertexIdx--next
+                            }
+                        }
+                        lastId = currentPoint;
+                        break;
+                    }
+                }
+            }
+        }
+
+        return sequence;
+    }
+
+    /**
+     * This method resets walker
+     *
+     * @param shuffle if TRUE, order of walks will be shuffled
+     */
+    @Override
+    public void reset(boolean shuffle) {
+        super.reset(shuffle);
+    }
+
+    public static class Builder<T extends SequenceElement> extends RandomWalker.Builder<T> {
+
+        public Builder(IGraph<T, ? extends Number> sourceGraph) {
+            super(sourceGraph);
+        }
+
+        /**
+         * This method specifies output sequence (walk) length
+         *
+         * @param walkLength
+         * @return
+         */
+        @Override
+        public Builder<T> setWalkLength(int walkLength) {
+            super.setWalkLength(walkLength);
+            return this;
+        }
+
+        /**
+         * This method defines walker behavior when it gets to node which has no next nodes available
+         * Default value: RESTART_ON_DISCONNECTED
+         *
+         * @param handling
+         * @return
+         */
+        @Override
+        public Builder<T> setNoEdgeHandling(@NonNull NoEdgeHandling handling) {
+            super.setNoEdgeHandling(handling);
+            return this;
+        }
+
+        /**
+         * This method specifies random seed.
+         *
+         * @param seed
+         * @return
+         */
+        @Override
+        public Builder<T> setSeed(long seed) {
+            super.setSeed(seed);
+            return this;
+        }
+
+        /**
+         * This method defines next hop selection within walk
+         *
+         * @param direction
+         * @return
+         */
+        @Override
+        public Builder<T> setWalkDirection(@NonNull WalkDirection direction) {
+            super.setWalkDirection(direction);
+            return this;
+        }
+
+        /**
+         * This method defines a chance for walk restart
+         * Good value would be somewhere between 0.03-0.07
+         *
+         * @param alpha
+         * @return
+         */
+        @Override
+        public RandomWalker.Builder<T> setRestartProbability(double alpha) {
+            return super.setRestartProbability(alpha);
+        }
+
+        public WeightedWalker<T> build() {
+            WeightedWalker<T> walker = new WeightedWalker<>();
+            walker.noEdgeHandling = this.noEdgeHandling;
+            walker.sourceGraph = this.sourceGraph;
+            walker.walkLength = this.walkLength;
+            walker.seed = this.seed;
+            walker.walkDirection = this.walkDirection;
+            walker.alpha = this.alpha;
+
+            walker.order = new int[sourceGraph.numVertices()];
+            for (int i = 0; i < walker.order.length; i++) {
+                walker.order[i] = i;
+            }
+
+            if (this.seed != 0)
+                walker.rng = new Random(this.seed);
+
+            return walker;
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/interfaces/SequenceElementFactory.java

@@ -0,0 +1,40 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.interfaces;
+
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+
+public interface SequenceElementFactory<T extends SequenceElement> {
+    /**
+     * This method builds object from provided JSON
+     *
+     * @param json JSON for restored object
+     * @return restored object
+     */
+    T deserialize(String json);
+
+    /**
+     * This method serializaes object  into JSON string
+     * @param element
+     * @return
+     */
+    String serialize(T element);
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/interfaces/SequenceIterator.java

@@ -0,0 +1,33 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.interfaces;
+
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+
+public interface SequenceIterator<T extends SequenceElement> {
+
+    boolean hasMoreSequences();
+
+    Sequence<T> nextSequence();
+
+    void reset();
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/interfaces/VectorsListener.java

@@ -0,0 +1,45 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.interfaces;
+
+import org.deeplearning4j.models.sequencevectors.SequenceVectors;
+import org.deeplearning4j.models.sequencevectors.enums.ListenerEvent;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+
+public interface VectorsListener<T extends SequenceElement> {
+
+    /**
+     * This method is called prior each processEvent call, to check if this specific VectorsListener implementation is viable for specific event
+     *
+     * @param event
+     * @param argument
+     * @return TRUE, if this event can and should be processed with this listener, FALSE otherwise
+     */
+    boolean validateEvent(ListenerEvent event, long argument);
+
+    /**
+     * This method is called at each epoch end
+     *
+     * @param event
+     * @param sequenceVectors
+     */
+    void processEvent(ListenerEvent event, SequenceVectors<T> sequenceVectors, long argument);
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/iterators/AbstractSequenceIterator.java

@@ -0,0 +1,94 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.iterators;
+
+import lombok.NonNull;
+import org.deeplearning4j.models.sequencevectors.interfaces.SequenceIterator;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+
+import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class AbstractSequenceIterator<T extends SequenceElement> implements SequenceIterator<T> {
+
+    private Iterable<Sequence<T>> underlyingIterable;
+    private Iterator<Sequence<T>> currentIterator;
+
+    // used to tag each sequence with own Id
+    protected AtomicInteger tagger = new AtomicInteger(0);
+
+    protected AbstractSequenceIterator(@NonNull Iterable<Sequence<T>> iterable) {
+        this.underlyingIterable = iterable;
+        this.currentIterator = iterable.iterator();
+    }
+
+    /**
+     * Checks, if there's more sequences available
+     * @return
+     */
+    @Override
+    public boolean hasMoreSequences() {
+        return currentIterator.hasNext();
+    }
+
+    /**
+     * Returns next sequence out of iterator
+     * @return
+     */
+    @Override
+    public Sequence<T> nextSequence() {
+        Sequence<T> sequence = currentIterator.next();
+        sequence.setSequenceId(tagger.getAndIncrement());
+        return sequence;
+    }
+
+    /**
+     * Resets iterator to first position
+     */
+    @Override
+    public void reset() {
+        tagger.set(0);
+        this.currentIterator = underlyingIterable.iterator();
+    }
+
+    public static class Builder<T extends SequenceElement> {
+        private Iterable<Sequence<T>> underlyingIterable;
+
+        /**
+         * Builds AbstractSequenceIterator on top of Iterable object
+         * @param iterable
+         */
+        public Builder(@NonNull Iterable<Sequence<T>> iterable) {
+            this.underlyingIterable = iterable;
+        }
+
+        /**
+         * Builds SequenceIterator
+         * @return
+         */
+        public AbstractSequenceIterator<T> build() {
+            AbstractSequenceIterator<T> iterator = new AbstractSequenceIterator<>(underlyingIterable);
+
+            return iterator;
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/iterators/FilteredSequenceIterator.java

@@ -0,0 +1,83 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.iterators;
+
+import lombok.NonNull;
+import org.deeplearning4j.models.sequencevectors.interfaces.SequenceIterator;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+
+public class FilteredSequenceIterator<T extends SequenceElement> implements SequenceIterator<T> {
+
+    private final SequenceIterator<T> underlyingIterator;
+    private final VocabCache<T> vocabCache;
+
+    /**
+     * Creates Filtered SequenceIterator on top of another SequenceIterator and appropriate VocabCache instance
+     *
+     * @param iterator
+     * @param vocabCache
+     */
+    public FilteredSequenceIterator(@NonNull SequenceIterator<T> iterator, @NonNull VocabCache<T> vocabCache) {
+        this.vocabCache = vocabCache;
+        this.underlyingIterator = iterator;
+    }
+
+    /**
+     * Checks, if there's any more sequences left in underlying iterator
+     * @return
+     */
+    @Override
+    public boolean hasMoreSequences() {
+        return underlyingIterator.hasMoreSequences();
+    }
+
+    /**
+     * Returns filtered sequence, that contains sequence elements from vocabulary only.
+     * Please note: it can return empty sequence, if no elements were found in vocabulary
+     * @return
+     */
+    @Override
+    public Sequence<T> nextSequence() {
+        Sequence<T> originalSequence = underlyingIterator.nextSequence();
+        Sequence<T> newSequence = new Sequence<>();
+
+        if (originalSequence != null)
+            for (T element : originalSequence.getElements()) {
+                if (element != null && vocabCache.hasToken(element.getLabel())) {
+                    newSequence.addElement(vocabCache.wordFor(element.getLabel()));
+                }
+            }
+
+        newSequence.setSequenceId(originalSequence.getSequenceId());
+
+        return newSequence;
+    }
+
+    /**
+     * Resets iterator down to first sequence
+     */
+    @Override
+    public void reset() {
+        underlyingIterator.reset();
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/iterators/SynchronizedSequenceIterator.java

@@ -0,0 +1,65 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.iterators;
+
+import lombok.NonNull;
+import org.deeplearning4j.models.sequencevectors.interfaces.SequenceIterator;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+
+public class SynchronizedSequenceIterator<T extends SequenceElement> implements SequenceIterator<T> {
+    protected SequenceIterator<T> underlyingIterator;
+
+    /**
+     * Creates SynchronizedSequenceIterator on top of any SequenceIterator
+     * @param iterator
+     */
+    public SynchronizedSequenceIterator(@NonNull SequenceIterator<T> iterator) {
+        this.underlyingIterator = iterator;
+    }
+
+    /**
+     * Checks, if there's any more sequences left in data source
+     * @return
+     */
+    @Override
+    public synchronized boolean hasMoreSequences() {
+        return underlyingIterator.hasMoreSequences();
+    }
+
+    /**
+     * Returns next sequence from data source
+     *
+     * @return
+     */
+    @Override
+    public synchronized Sequence<T> nextSequence() {
+        return underlyingIterator.nextSequence();
+    }
+
+    /**
+     * This method resets underlying iterator
+     */
+    @Override
+    public synchronized void reset() {
+        underlyingIterator.reset();
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/listeners/ScoreListener.java

@@ -0,0 +1,63 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.listeners;
+
+import lombok.NonNull;
+import org.deeplearning4j.models.sequencevectors.SequenceVectors;
+import org.deeplearning4j.models.sequencevectors.enums.ListenerEvent;
+import org.deeplearning4j.models.sequencevectors.interfaces.VectorsListener;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+@Deprecated
+public class ScoreListener<T extends SequenceElement> implements VectorsListener<T> {
+    protected static final Logger logger = LoggerFactory.getLogger(ScoreListener.class);
+    private final ListenerEvent targetEvent;
+    private final AtomicLong callsCount = new AtomicLong(0);
+    private final int frequency;
+
+    public ScoreListener(@NonNull ListenerEvent targetEvent, int frequency) {
+        this.targetEvent = targetEvent;
+        this.frequency = frequency;
+    }
+
+    @Override
+    public boolean validateEvent(ListenerEvent event, long argument) {
+        if (event == targetEvent)
+            return true;
+
+        return false;
+    }
+
+    @Override
+    public void processEvent(ListenerEvent event, SequenceVectors<T> sequenceVectors, long argument) {
+        if (event != targetEvent)
+            return;
+
+        callsCount.incrementAndGet();
+
+        if (callsCount.get() % frequency == 0)
+            logger.info("Average score for last batch: {}", sequenceVectors.getElementsScore());
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/listeners/SerializingListener.java

@@ -0,0 +1,167 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.listeners;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.models.sequencevectors.SequenceVectors;
+import org.deeplearning4j.models.sequencevectors.enums.ListenerEvent;
+import org.deeplearning4j.models.sequencevectors.interfaces.VectorsListener;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.nd4j.common.util.SerializationUtils;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.concurrent.Semaphore;
+
+@Slf4j
+public class SerializingListener<T extends SequenceElement> implements VectorsListener<T> {
+    private File targetFolder = new File("./");
+    private String modelPrefix = "Model_";
+    private boolean useBinarySerialization = true;
+    private ListenerEvent targetEvent = ListenerEvent.EPOCH;
+    private int targetFrequency = 100000;
+
+    private Semaphore locker = new Semaphore(1);
+
+    protected SerializingListener() {}
+
+    /**
+     * This method is called prior each processEvent call, to check if this specific VectorsListener implementation is viable for specific event
+     *
+     * @param event
+     * @param argument
+     * @return TRUE, if this event can and should be processed with this listener, FALSE otherwise
+     */
+    @Override
+    public boolean validateEvent(ListenerEvent event, long argument) {
+        try {
+            /**
+             * please note, since sequence vectors are multithreaded we need to stop processed while model is being saved
+             */
+            locker.acquire();
+
+            if (event == targetEvent && argument % targetFrequency == 0) {
+                return true;
+            } else
+                return false;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        } finally {
+            locker.release();
+        }
+    }
+
+    /**
+     * This method is called at each epoch end
+     *
+     * @param event
+     * @param sequenceVectors
+     * @param argument
+     */
+    @Override
+    public void processEvent(ListenerEvent event, SequenceVectors<T> sequenceVectors, long argument) {
+        try {
+            locker.acquire();
+
+            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
+
+            StringBuilder builder = new StringBuilder(targetFolder.getAbsolutePath());
+            builder.append("/").append(modelPrefix).append("_").append(sdf.format(new Date())).append(".seqvec");
+            File targetFile = new File(builder.toString());
+
+            if (useBinarySerialization) {
+                SerializationUtils.saveObject(sequenceVectors, targetFile);
+            } else {
+                throw new UnsupportedOperationException("Not implemented yet");
+            }
+
+        } catch (Exception e) {
+            log.error("",e);
+        } finally {
+            locker.release();
+        }
+    }
+
+    public static class Builder<T extends SequenceElement> {
+        private File targetFolder = new File("./");
+        private String modelPrefix = "Model_";
+        private boolean useBinarySerialization = true;
+        private ListenerEvent targetEvent = ListenerEvent.EPOCH;
+        private int targetFrequency = 100000;
+
+        public Builder(ListenerEvent targetEvent, int frequency) {
+            this.targetEvent = targetEvent;
+            this.targetFrequency = frequency;
+        }
+
+        /**
+         * This method allows you to define template for file names that will be created during serialization
+         * @param reallyUse
+         * @return
+         */
+        public Builder<T> setFilenamePrefix(boolean reallyUse) {
+            this.useBinarySerialization = reallyUse;
+            return this;
+        }
+
+        /**
+         * This method specifies target folder where models should be saved
+         *
+         * @param folder
+         * @return
+         */
+        public Builder<T> setTargetFolder(@NonNull String folder) {
+            this.setTargetFolder(new File(folder));
+            return this;
+        }
+
+        /**
+         * This method specifies target folder where models should be saved
+         *
+         * @param folder
+         * @return
+         */
+        public Builder<T> setTargetFolder(@NonNull File folder) {
+            if (!folder.exists() || !folder.isDirectory())
+                throw new IllegalStateException("Target folder must exist!");
+            this.targetFolder = folder;
+            return this;
+        }
+
+        /**
+         * This method returns new SerializingListener instance
+         *
+         * @return
+         */
+        public SerializingListener<T> build() {
+            SerializingListener<T> listener = new SerializingListener<>();
+            listener.modelPrefix = this.modelPrefix;
+            listener.targetFolder = this.targetFolder;
+            listener.useBinarySerialization = this.useBinarySerialization;
+            listener.targetEvent = this.targetEvent;
+            listener.targetFrequency = this.targetFrequency;
+
+            return listener;
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/listeners/SimilarityListener.java

@@ -0,0 +1,66 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.listeners;
+
+import org.deeplearning4j.models.sequencevectors.SequenceVectors;
+import org.deeplearning4j.models.sequencevectors.enums.ListenerEvent;
+import org.deeplearning4j.models.sequencevectors.interfaces.VectorsListener;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+public class SimilarityListener<T extends SequenceElement> implements VectorsListener<T> {
+    protected static final Logger logger = LoggerFactory.getLogger(SimilarityListener.class);
+    private final ListenerEvent targetEvent;
+    private final int frequency;
+    private final String element1;
+    private final String element2;
+    private final AtomicLong counter = new AtomicLong(0);
+
+    public SimilarityListener(ListenerEvent targetEvent, int frequency, String label1, String label2) {
+        this.targetEvent = targetEvent;
+        this.frequency = frequency;
+        this.element1 = label1;
+        this.element2 = label2;
+    }
+
+    @Override
+    public boolean validateEvent(ListenerEvent event, long argument) {
+        return event == targetEvent;
+    }
+
+    @Override
+    public void processEvent(ListenerEvent event, SequenceVectors<T> sequenceVectors, long argument) {
+        if (event != targetEvent)
+            return;
+
+        long cnt = counter.getAndIncrement();
+
+        if (cnt % frequency != 0)
+            return;
+
+        double similarity = sequenceVectors.similarity(element1, element2);
+
+        logger.info("Invocation: {}, similarity: {}", cnt, similarity);
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/Sequence.java

@@ -0,0 +1,237 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.sequence;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+
+import java.io.Serializable;
+import java.util.*;
+
+public class Sequence<T extends SequenceElement> implements Serializable {
+
+    private static final long serialVersionUID = 2223750736522624735L;
+
+    protected List<T> elements = new ArrayList<>();
+
+    // elements map needed to speedup searches against elements in sequence
+    protected Map<String, T> elementsMap = new LinkedHashMap<>();
+
+    // each document can have multiple labels
+    protected List<T> labels = new ArrayList<>();
+
+    protected T label;
+
+    protected int hash = 0;
+    protected boolean hashCached = false;
+
+    @Getter
+    @Setter
+    protected int sequenceId;
+
+    /**
+     * Creates new empty sequence
+     *
+     */
+    public Sequence() {
+
+    }
+
+    /**
+     * Creates new sequence from collection of elements
+     *
+     * @param set
+     */
+    public Sequence(@NonNull Collection<T> set) {
+        this();
+        addElements(set);
+    }
+
+    /**
+     * Adds single element to sequence
+     *
+     * @param element
+     */
+    public synchronized void addElement(@NonNull T element) {
+        hashCached = false;
+        this.elementsMap.put(element.getLabel(), element);
+        this.elements.add(element);
+    }
+
+    /**
+     * Adds collection of elements to the sequence
+     *
+     * @param set
+     */
+    public void addElements(Collection<T> set) {
+        for (T element : set) {
+            addElement(element);
+        }
+    }
+
+    /**
+     * Returns this sequence as list of labels
+     * @return
+     */
+    public List<String> asLabels() {
+        List<String> labels = new ArrayList<>();
+        for (T element : getElements()) {
+            labels.add(element.getLabel());
+        }
+        return labels;
+    }
+
+    /**
+     * Returns single element out of this sequence by its label
+     *
+     * @param label
+     * @return
+     */
+    public T getElementByLabel(@NonNull String label) {
+        return elementsMap.get(label);
+    }
+
+    /**
+     * Returns an ordered unmodifiable list of elements from this sequence
+     *
+     * @return
+     */
+    public List<T> getElements() {
+        return Collections.unmodifiableList(elements);
+    }
+
+    /**
+     * Returns label for this sequence
+     *
+     * @return label for this sequence, null if label was not defined
+     */
+    public T getSequenceLabel() {
+        return label;
+    }
+
+    /**
+     * Returns all labels for this sequence
+     *
+     * @return
+     */
+    public List<T> getSequenceLabels() {
+        return labels;
+    }
+
+    /**
+     * Sets sequence labels
+     * @param labels
+     */
+    public void setSequenceLabels(List<T> labels) {
+        this.labels = labels;
+    }
+
+    /**
+     * Set sequence label
+     *
+     * @param label
+     */
+    public void setSequenceLabel(@NonNull T label) {
+        this.label = label;
+        if (!labels.contains(label))
+            labels.add(label);
+    }
+
+    /**
+     *  Adds sequence label. In this case sequence will have multiple labels
+     *
+     * @param label
+     */
+    public void addSequenceLabel(@NonNull T label) {
+        this.labels.add(label);
+        if (this.label == null)
+            this.label = label;
+    }
+
+    /**
+     * Checks, if sequence is empty
+     *
+     * @return TRUE if empty, FALSE otherwise
+     */
+    public boolean isEmpty() {
+        return elements.isEmpty();
+    }
+
+    /**
+     * This method returns number of elements in this sequence
+     *
+     * @return
+     */
+    public int size() {
+        return elements.size();
+    }
+
+    /**
+     * This method returns  sequence element by index
+     *
+     * @param index
+     * @return
+     */
+    public T getElementByIndex(int index) {
+        return elements.get(index);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        Sequence<?> sequence = (Sequence<?>) o;
+
+        return elements != null ? elements.equals(sequence.elements) : sequence.elements == null;
+
+    }
+
+    @Override
+    public int hashCode() {
+        if (hashCached)
+            return hash;
+
+        for (T element : elements) {
+            hash += 31 * element.hashCode();
+        }
+
+        hashCached = true;
+
+        return hash;
+    }
+
+    @Override
+    public String toString() {
+        return "Sequence{" +
+                "elements=" + elements +
+                ", elementsMap=" + elementsMap +
+                ", labels=" + labels +
+                ", label=" + label +
+                ", hash=" + hash +
+                ", hashCached=" + hashCached +
+                ", sequenceId=" + sequenceId +
+                '}';
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/SequenceElement.java

@@ -0,0 +1,382 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.sequence;
+
+import org.nd4j.shade.guava.util.concurrent.AtomicDouble;
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+import org.deeplearning4j.models.word2vec.VocabWord;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.util.HashUtil;
+import org.nd4j.shade.jackson.annotation.*;
+import org.nd4j.shade.jackson.databind.DeserializationFeature;
+import org.nd4j.shade.jackson.databind.MapperFeature;
+import org.nd4j.shade.jackson.databind.ObjectMapper;
+import org.nd4j.shade.jackson.databind.SerializationFeature;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class")
+@JsonSubTypes({
+        @JsonSubTypes.Type(value = VocabWord.class, name = "vocabWord")
+})
+@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE,
+        setterVisibility = JsonAutoDetect.Visibility.NONE)
+public abstract class SequenceElement implements Comparable<SequenceElement>, Serializable {
+
+    private static final long serialVersionUID = 2223750736522624732L;
+
+    @JsonProperty
+    protected AtomicDouble elementFrequency = new AtomicDouble(0);
+
+    //used in comparison when building the huffman tree
+    protected int index = -1;
+    protected List<Byte> codes = new ArrayList<>();
+
+    protected List<Integer> points = new ArrayList<>();
+    protected short codeLength = 0;
+
+    // this var defines, if this token can't be truncated with minWordFrequency threshold
+    @Getter
+    @Setter
+    protected boolean special;
+
+    // this var defines that we have label here
+    @Getter
+    @Setter
+    protected boolean isLabel;
+
+    // this var defines how many documents/sequences contain this word
+    protected AtomicLong sequencesCount = new AtomicLong(0);
+
+
+    // this var is used as state for preciseWeightInit routine, to avoid multiple initializations for the same data
+    @Getter
+    @Setter
+    protected boolean init;
+
+    /*
+            Reserved for Joint/Distributed vocabs mechanics
+    */
+    @Setter
+    protected Long storageId;
+
+    @Getter
+    @Setter
+    protected boolean isLocked;
+
+    /**
+     * This method should return string representation of this SequenceElement, so it can be used for
+     *
+     * @return
+     */
+    abstract public String getLabel();
+
+    /**
+     * This method returns number of documents/sequences where this element was evidenced
+     *
+     * @return
+     */
+    public long getSequencesCount() {
+        return sequencesCount.get();
+    }
+
+    /**
+     * This method sets documents count to specified value
+     *
+     * @param count
+     */
+    public void setSequencesCount(long count) {
+        this.sequencesCount.set(count);
+    }
+
+    /**
+     * Increments document count by one
+     */
+    public void incrementSequencesCount() {
+        this.sequencesCount.incrementAndGet();
+    }
+
+    /**
+     * Increments document count by specified value
+     * @param count
+     */
+    public void incrementSequencesCount(long count) {
+        this.sequencesCount.addAndGet(count);
+    }
+
+    /**
+     * Returns whether this element was defined as label, or no
+     *
+     * @return
+     */
+    public boolean isLabel() {
+        return isLabel;
+    }
+
+    /**
+     * This method specifies, whether this element should be treated as label for some sequence/document or not.
+     *
+     * @param isLabel
+     */
+    public void markAsLabel(boolean isLabel) {
+        this.isLabel = isLabel;
+    }
+
+    /**
+     * This method returns SequenceElement's frequency in current training corpus.
+     *
+     * @return
+     */
+    public double getElementFrequency() {
+        return elementFrequency.get();
+    }
+
+    /**
+     * This method sets frequency value for this element
+     *
+     * @param value
+     */
+    public void setElementFrequency(long value) {
+        elementFrequency.set(value);
+    }
+
+    /**
+     * Increases element frequency counter by 1
+     */
+    public void incrementElementFrequency() {
+        increaseElementFrequency(1);
+    }
+
+    /**
+     * Increases element frequency counter by argument
+     *
+     * @param by
+     */
+    public void increaseElementFrequency(int by) {
+        elementFrequency.getAndAdd(by);
+    }
+
+    /**
+     * Equals method override should be properly implemented for any extended class, otherwise it will be based on label equality
+     *
+     * @param object
+     * @return
+     */
+    public boolean equals(Object object) {
+        if (this == object)
+            return true;
+        if (object == null)
+            return false;
+        if (!(object instanceof SequenceElement))
+            return false;
+
+        return this.getLabel().equals(((SequenceElement) object).getLabel());
+    }
+
+    /**
+     *  Returns index in Huffman tree
+     *
+     * @return index >= 0, if tree was built, -1 otherwise
+     */
+    public int getIndex() {
+        return index;
+    }
+
+    /**
+     * Sets index in Huffman tree
+     *
+     * @param index
+     */
+    public void setIndex(int index) {
+        this.index = index;
+    }
+
+    /**
+     * Returns Huffman tree codes
+     * @return
+     */
+    public List<Byte> getCodes() {
+        return codes;
+    }
+
+    /**
+     * Sets Huffman tree codes
+     * @param codes
+     */
+    public void setCodes(List<Byte> codes) {
+        this.codes = codes;
+    }
+
+    /**
+     * Returns Huffman tree points
+     *
+     * @return
+     */
+    public List<Integer> getPoints() {
+        return points;
+    }
+
+    /**
+     * Sets Huffman tree points
+     *
+     * @param points
+     */
+    public void setPoints(List<Integer> points) {
+        this.points = points;
+    }
+
+    /**
+     * Sets Huffman tree points
+     *
+     * @param points
+     */
+    @JsonIgnore
+    public void setPoints(int[] points) {
+        this.points = new ArrayList<>();
+        for (int i = 0; i < points.length; i++) {
+            this.points.add(points[i]);
+        }
+    }
+
+    /**
+     * Returns Huffman code length.
+     *
+     * Please note: maximum vocabulary/tree size depends on code length
+     *
+     * @return
+     */
+    public int getCodeLength() {
+        return codeLength;
+    }
+
+    /**
+     * This method fills codes and points up to codeLength
+     *
+     * @param codeLength
+     */
+    public void setCodeLength(short codeLength) {
+        this.codeLength = codeLength;
+        if (codes.size() < codeLength) {
+            for (int i = 0; i < codeLength; i++)
+                codes.add((byte) 0);
+        }
+
+        if (points.size() < codeLength) {
+            for (int i = 0; i < codeLength; i++)
+                points.add(0);
+        }
+    }
+
+
+    public static final long getLongHash(@NonNull String string) {
+        return HashUtil.getLongHash(string);
+    }
+
+    /**
+     * Returns gradient for this specific element, at specific position
+     * @param index
+     * @param g
+     * @param lr
+     * @return
+     */
+    @Deprecated
+    public double getGradient(int index, double g, double lr) {
+        /*
+        if (adaGrad == null)
+            adaGrad = new AdaGrad(1,getCodeLength(), lr);
+
+        return adaGrad.getGradient(g, index, new int[]{1, getCodeLength()});
+        */
+        return 0.0;
+    }
+
+    @Deprecated
+    public void setHistoricalGradient(INDArray gradient) {
+        /*
+        if (adaGrad == null)
+            adaGrad = new AdaGrad(1,getCodeLength(), 0.025);
+
+        adaGrad.setHistoricalGradient(gradient);
+        */
+    }
+
+    @Deprecated
+    public INDArray getHistoricalGradient() {
+        /*
+        if (adaGrad == null)
+            adaGrad = new AdaGrad(1,getCodeLength(), 0.025);
+        return adaGrad.getHistoricalGradient();
+        */
+        return null;
+    }
+
+    /**
+     * hashCode method override should be properly implemented for any extended class, otherwise it will be based on label hashCode
+     *
+     * @return hashCode for this SequenceElement
+     */
+    public int hashCode() {
+        if (this.getLabel() == null)
+            throw new IllegalStateException("Label should not be null");
+        return this.getLabel().hashCode();
+    }
+
+    @Override
+    public int compareTo(SequenceElement o) {
+        return Double.compare(elementFrequency.get(), o.elementFrequency.get());
+    }
+
+    @Override
+    public String toString() {
+        return "SequenceElement: {label: '" + this.getLabel() + "'," + " freq: '" + elementFrequency.get() + "',"
+                        + " codes: " + codes.toString() + " points: " + points.toString() + " index: '" + this.index
+                        + "'}";
+    }
+
+    /**
+     *
+     * @return
+     */
+    public abstract String toJSON();
+
+    public Long getStorageId() {
+        if (storageId == null)
+            storageId = SequenceElement.getLongHash(this.getLabel());
+        return storageId;
+    }
+
+    public static ObjectMapper mapper() {
+        /*
+              DO NOT ENABLE INDENT_OUTPUT FEATURE
+              we need THIS json to be single-line
+          */
+        ObjectMapper ret = new ObjectMapper();
+        ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
+        ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true);
+        return ret;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/ShallowSequenceElement.java

@@ -0,0 +1,39 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.sequence;
+
+public class ShallowSequenceElement extends SequenceElement {
+
+    public ShallowSequenceElement(double frequency, long id) {
+        this.storageId = id;
+        this.elementFrequency.set(frequency);
+    }
+
+    @Override
+    public String getLabel() {
+        return null;
+    }
+
+    @Override
+    public String toJSON() {
+        return null;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/serialization/AbstractElementFactory.java

@@ -0,0 +1,88 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.serialization;
+
+import lombok.NonNull;
+import org.deeplearning4j.models.sequencevectors.interfaces.SequenceElementFactory;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.nd4j.shade.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class AbstractElementFactory<T extends SequenceElement> implements SequenceElementFactory<T> {
+    private final Class targetClass;
+
+    protected static final Logger log = LoggerFactory.getLogger(AbstractElementFactory.class);
+
+    /**
+     * This is the only constructor available for AbstractElementFactory
+     * @param cls class that going to be serialized/deserialized using this instance. I.e.: VocabWord.class
+     */
+    public AbstractElementFactory(@NonNull Class<? extends SequenceElement> cls) {
+        targetClass = cls;
+    }
+
+    /**
+     * This method builds object from provided JSON
+     *
+     * @param json JSON for restored object
+     * @return restored object
+     */
+    @Override
+    public T deserialize(String json) {
+        ObjectMapper mapper = SequenceElement.mapper();
+        try {
+            T ret = (T) mapper.readValue(json, targetClass);
+            return ret;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * This method serializaes object  into JSON string
+     *
+     * @param element
+     * @return
+     */
+    @Override
+    public String serialize(T element) {
+        String json = null;
+        try {
+            json = element.toJSON();
+        } catch (Exception e) {
+            log.error("Direct serialization failed, falling back to jackson");
+        }
+
+        if (json == null || json.isEmpty()) {
+            ObjectMapper mapper = SequenceElement.mapper();
+            try {
+                json = mapper.writeValueAsString(element);
+            } catch (org.nd4j.shade.jackson.core.JsonProcessingException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        return json;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/serialization/VocabWordFactory.java

@@ -0,0 +1,59 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.serialization;
+
+import org.deeplearning4j.models.sequencevectors.interfaces.SequenceElementFactory;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.VocabWord;
+import org.nd4j.shade.jackson.databind.ObjectMapper;
+
+import java.io.IOException;
+
+public class VocabWordFactory implements SequenceElementFactory<VocabWord> {
+
+    /**
+     * This method builds object from provided JSON
+     *
+     * @param json JSON for restored object
+     * @return restored object
+     */
+    @Override
+    public VocabWord deserialize(String json) {
+        ObjectMapper mapper = SequenceElement.mapper();
+        try {
+            VocabWord ret = mapper.readValue(json, VocabWord.class);
+            return ret;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * This method serializaes object  into JSON string
+     *
+     * @param element
+     * @return
+     */
+    @Override
+    public String serialize(VocabWord element) {
+        return element.toJSON();
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/SequenceTransformer.java

@@ -0,0 +1,47 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.transformers;
+
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+
+public interface SequenceTransformer<T extends SequenceElement, V extends Object> {
+
+    /**
+     * Returns Vocabulary derived from underlying data source.
+     * In default implementations this method heavily relies on transformToSequence() method.
+     *
+     * @return
+     */
+    //VocabCache<T> derivedVocabulary();
+
+    /**
+     * This is generic method for transformation data from any format to Sequence of SequenceElement.
+     * It will be used both in Vocab building, and in training process
+     *
+     * @param object - Object to be transformed into Sequence
+     * @return
+     */
+    Sequence<T> transformToSequence(V object);
+
+
+    void reset();
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/GraphTransformer.java

@@ -0,0 +1,178 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.transformers.impl;
+
+import lombok.NonNull;
+import org.deeplearning4j.models.sequencevectors.graph.primitives.IGraph;
+import org.deeplearning4j.models.sequencevectors.graph.walkers.GraphWalker;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.Huffman;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.deeplearning4j.text.labels.LabelsProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class GraphTransformer<T extends SequenceElement> implements Iterable<Sequence<T>> {
+    protected IGraph<T, ?> sourceGraph;
+    protected GraphWalker<T> walker;
+    protected LabelsProvider<T> labelsProvider;
+    protected AtomicInteger counter = new AtomicInteger(0);
+    protected boolean shuffle = true;
+    protected VocabCache<T> vocabCache;
+
+    protected static final Logger log = LoggerFactory.getLogger(GraphTransformer.class);
+
+    protected GraphTransformer() {}
+
+    /**
+     * This method handles required initialization for GraphTransformer
+     */
+    protected void initialize() {
+        log.info("Building Huffman tree for source graph...");
+        int nVertices = sourceGraph.numVertices();
+        //int[] degrees = new int[nVertices];
+        //for( int i=0; i<nVertices; i++ )
+        // degrees[i] = sourceGraph.getVertexDegree(i);
+        /*
+        for (int y = 0; y < nVertices; y+= 20) {
+            int[] copy = Arrays.copyOfRange(degrees, y, y+20);
+            System.out.println("D: " + Arrays.toString(copy));
+        }
+        */
+        //        GraphHuffman huffman = new GraphHuffman(nVertices);
+        //        huffman.buildTree(degrees);
+
+        log.info("Transferring Huffman tree info to nodes...");
+        for (int i = 0; i < nVertices; i++) {
+            T element = sourceGraph.getVertex(i).getValue();
+            element.setElementFrequency(sourceGraph.getConnectedVertices(i).size());
+
+            if (vocabCache != null)
+                vocabCache.addToken(element);
+        }
+
+        if (vocabCache != null) {
+            Huffman huffman = new Huffman(vocabCache.vocabWords());
+            huffman.build();
+            huffman.applyIndexes(vocabCache);
+        }
+    }
+
+
+    @Override
+    public Iterator<Sequence<T>> iterator() {
+        this.counter.set(0);
+        this.walker.reset(shuffle);
+        return new Iterator<Sequence<T>>() {
+            private GraphWalker<T> walker = GraphTransformer.this.walker;
+
+            @Override
+            public void remove() {
+                throw new UnsupportedOperationException("This is not supported on read-only iterator");
+            }
+
+            @Override
+            public boolean hasNext() {
+                return walker.hasNext();
+            }
+
+            @Override
+            public Sequence<T> next() {
+                Sequence<T> sequence = walker.next();
+                sequence.setSequenceId(counter.getAndIncrement());
+
+                // we might already have labels defined from walker
+                if (walker.isLabelEnabled() && sequence.getSequenceLabels() == null)
+                    if (labelsProvider != null) {
+                        // TODO: sequence labels to be implemented for graph walks
+                        sequence.setSequenceLabel(labelsProvider.getLabel(sequence.getSequenceId()));
+                    }
+
+                return sequence;
+            }
+        };
+    }
+
+    public static class Builder<T extends SequenceElement> {
+        protected IGraph<T, ?> sourceGraph;
+        protected LabelsProvider<T> labelsProvider;
+        protected GraphWalker<T> walker;
+        protected boolean shuffle = true;
+        protected VocabCache<T> vocabCache;
+
+        public Builder() {
+            //
+        }
+
+        public Builder(@NonNull GraphWalker<T> walker) {
+            this.walker = walker;
+        }
+
+        public Builder(@NonNull IGraph<T, ?> sourceGraph) {
+            this.sourceGraph = sourceGraph;
+        }
+
+
+        public Builder<T> setLabelsProvider(@NonNull LabelsProvider<T> provider) {
+            this.labelsProvider = provider;
+            return this;
+        }
+
+        public Builder<T> setGraphWalker(@NonNull GraphWalker<T> walker) {
+            this.walker = walker;
+            return this;
+        }
+
+        public Builder<T> setVocabCache(@NonNull VocabCache<T> vocabCache) {
+            this.vocabCache = vocabCache;
+            return this;
+        }
+
+        public Builder<T> shuffleOnReset(boolean reallyShuffle) {
+            this.shuffle = reallyShuffle;
+            return this;
+        }
+
+        public GraphTransformer<T> build() {
+            if (this.walker == null)
+                throw new IllegalStateException("Please provide GraphWalker instance.");
+
+            GraphTransformer<T> transformer = new GraphTransformer<>();
+            if (this.sourceGraph == null)
+                this.sourceGraph = walker.getSourceGraph();
+
+            transformer.sourceGraph = this.sourceGraph;
+            transformer.labelsProvider = this.labelsProvider;
+            transformer.shuffle = this.shuffle;
+            transformer.vocabCache = this.vocabCache;
+            transformer.walker = this.walker;
+
+            // FIXME: get rid of this
+            transformer.initialize();
+
+            return transformer;
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/SentenceTransformer.java

@@ -0,0 +1,161 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.transformers.impl;
+
+import lombok.NonNull;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.transformers.SequenceTransformer;
+import org.deeplearning4j.models.sequencevectors.transformers.impl.iterables.BasicTransformerIterator;
+import org.deeplearning4j.models.sequencevectors.transformers.impl.iterables.ParallelTransformerIterator;
+import org.deeplearning4j.models.word2vec.VocabWord;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.deeplearning4j.text.documentiterator.BasicLabelAwareIterator;
+import org.deeplearning4j.text.documentiterator.DocumentIterator;
+import org.deeplearning4j.text.documentiterator.LabelAwareIterator;
+import org.deeplearning4j.text.sentenceiterator.SentenceIterator;
+import org.deeplearning4j.text.tokenization.tokenizer.Tokenizer;
+import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class SentenceTransformer implements SequenceTransformer<VocabWord, String>, Iterable<Sequence<VocabWord>> {
+    /*
+            So, we must accept any SentenceIterator implementations, and build vocab out of it, and use it for further transforms between text and Sequences
+     */
+    protected TokenizerFactory tokenizerFactory;
+    protected LabelAwareIterator iterator;
+    protected boolean readOnly = false;
+    protected AtomicInteger sentenceCounter = new AtomicInteger(0);
+    protected boolean allowMultithreading = false;
+    protected BasicTransformerIterator currentIterator;
+
+    protected static final Logger log = LoggerFactory.getLogger(SentenceTransformer.class);
+    private VocabCache<VocabWord> vocabCache;
+
+    private SentenceTransformer(@NonNull LabelAwareIterator iterator) {
+        this.iterator = iterator;
+    }
+
+    @Override
+    public Sequence<VocabWord> transformToSequence(String object) {
+        Sequence<VocabWord> sequence = new Sequence<>();
+
+        Tokenizer tokenizer = tokenizerFactory.create(object);
+        List<String> list = tokenizer.getTokens();
+
+        for (String token : list) {
+            if (token == null || token.isEmpty() || token.trim().isEmpty())
+                continue;
+
+            VocabWord word = vocabCache.wordFor(token);
+            if(word == null)
+                word = new VocabWord(1.0, token);
+            sequence.addElement(word);
+        }
+
+        sequence.setSequenceId(sentenceCounter.getAndIncrement());
+        return sequence;
+    }
+
+    @Override
+    public Iterator<Sequence<VocabWord>> iterator() {
+        if (currentIterator == null) {
+            currentIterator = new BasicTransformerIterator(iterator, this);
+
+        } else
+            reset();
+
+        return currentIterator;
+    }
+
+    @Override
+    public void reset() {
+        if (currentIterator != null)
+            currentIterator.reset();
+    }
+
+
+    public static class Builder {
+        protected TokenizerFactory tokenizerFactory;
+        protected LabelAwareIterator iterator;
+        protected VocabCache<VocabWord> vocabCache;
+        protected boolean readOnly = false;
+        protected boolean allowMultithreading = false;
+
+        public Builder() {
+
+        }
+
+        public Builder vocabCache(@NonNull VocabCache<VocabWord> vocabCache) {
+            this.vocabCache = vocabCache;
+            return this;
+        }
+
+        public Builder tokenizerFactory(@NonNull TokenizerFactory tokenizerFactory) {
+            this.tokenizerFactory = tokenizerFactory;
+            return this;
+        }
+
+        public Builder iterator(@NonNull LabelAwareIterator iterator) {
+            this.iterator = iterator;
+            return this;
+        }
+
+        public Builder iterator(@NonNull SentenceIterator iterator) {
+            this.iterator = new BasicLabelAwareIterator.Builder(iterator).build();
+            return this;
+        }
+
+        public Builder iterator(@NonNull DocumentIterator iterator) {
+            this.iterator = new BasicLabelAwareIterator.Builder(iterator).build();
+            return this;
+        }
+
+        public Builder readOnly(boolean readOnly) {
+            this.readOnly = true;
+            return this;
+        }
+
+        /**
+         * This method enables/disables parallel processing over sentences
+         *
+         * @param reallyAllow
+         * @return
+         */
+        public Builder allowMultithreading(boolean reallyAllow) {
+            this.allowMultithreading = reallyAllow;
+            return this;
+        }
+
+        public SentenceTransformer build() {
+            SentenceTransformer transformer = new SentenceTransformer(this.iterator);
+            transformer.tokenizerFactory = this.tokenizerFactory;
+            transformer.readOnly = this.readOnly;
+            transformer.allowMultithreading = this.allowMultithreading;
+            transformer.vocabCache = this.vocabCache;
+            return transformer;
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/BasicTransformerIterator.java

@@ -0,0 +1,78 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.transformers.impl.iterables;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.transformers.impl.SentenceTransformer;
+import org.deeplearning4j.models.word2vec.VocabWord;
+import org.deeplearning4j.text.documentiterator.LabelAwareIterator;
+import org.deeplearning4j.text.documentiterator.LabelledDocument;
+
+import java.util.Iterator;
+
+@Slf4j
+public class BasicTransformerIterator implements Iterator<Sequence<VocabWord>> {
+    protected final LabelAwareIterator iterator;
+    protected boolean allowMultithreading;
+    protected final SentenceTransformer sentenceTransformer;
+
+    public BasicTransformerIterator(@NonNull LabelAwareIterator iterator, @NonNull SentenceTransformer transformer) {
+        this.iterator = iterator;
+        this.allowMultithreading = false;
+        this.sentenceTransformer = transformer;
+
+        this.iterator.reset();
+    }
+
+    @Override
+    public boolean hasNext() {
+        return this.iterator.hasNextDocument();
+    }
+
+    @Override
+    public Sequence<VocabWord> next() {
+        LabelledDocument document = iterator.nextDocument();
+        if (document == null || document.getContent() == null)
+            return new Sequence<>();
+        Sequence<VocabWord> sequence = sentenceTransformer.transformToSequence(document.getContent());
+
+        if (document.getLabels() != null)
+            for (String label : document.getLabels()) {
+                if (label != null && !label.isEmpty())
+                    sequence.addSequenceLabel(new VocabWord(1.0, label));
+            }
+
+
+        return sequence;
+    }
+
+
+    public void reset() {
+        this.iterator.reset();
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException();
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/ParallelTransformerIterator.java

@@ -0,0 +1,174 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.sequencevectors.transformers.impl.iterables;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.transformers.impl.SentenceTransformer;
+import org.deeplearning4j.models.word2vec.VocabWord;
+import org.deeplearning4j.text.documentiterator.AsyncLabelAwareIterator;
+import org.deeplearning4j.text.documentiterator.LabelAwareIterator;
+import org.deeplearning4j.text.documentiterator.LabelledDocument;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+@Slf4j
+public class ParallelTransformerIterator extends BasicTransformerIterator {
+
+    protected static final int capacity = 1024;
+    protected BlockingQueue<Future<Sequence<VocabWord>>> buffer = new LinkedBlockingQueue<>(capacity);
+    //protected BlockingQueue<LabelledDocument> stringBuffer;
+    //protected TokenizerThread[] threads;
+    protected AtomicBoolean underlyingHas = new AtomicBoolean(true);
+    protected AtomicInteger processing = new AtomicInteger(0);
+
+    private ExecutorService executorService;
+
+    protected static final AtomicInteger count = new AtomicInteger(0);
+
+    private static final int PREFETCH_SIZE = 100;
+
+    public ParallelTransformerIterator(@NonNull LabelAwareIterator iterator, @NonNull SentenceTransformer transformer) {
+        this(iterator, transformer, true);
+    }
+
+    private void prefetchIterator() {
+        /*for (int i = 0; i < PREFETCH_SIZE; ++i) {
+            //boolean before = underlyingHas;
+
+                if (underlyingHas.get())
+                    underlyingHas.set(iterator.hasNextDocument());
+                else
+                    underlyingHas.set(false);
+
+            if (underlyingHas.get()) {
+                CallableTransformer callableTransformer = new CallableTransformer(iterator.nextDocument(), sentenceTransformer);
+                Future<Sequence<VocabWord>> futureSequence = executorService.submit(callableTransformer);
+                try {
+                    buffer.put(futureSequence);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+        }*/
+    }
+
+    public ParallelTransformerIterator(@NonNull LabelAwareIterator iterator, @NonNull SentenceTransformer transformer,
+                                       boolean allowMultithreading) {
+        super(new AsyncLabelAwareIterator(iterator, 512), transformer);
+        //super(iterator, transformer);
+        this.allowMultithreading = allowMultithreading;
+        //this.stringBuffer = new LinkedBlockingQueue<>(512);
+
+        //threads = new TokenizerThread[1];
+        //threads = new TokenizerThread[allowMultithreading ? Math.max(Runtime.getRuntime().availableProcessors(), 2) : 1];
+        executorService = Executors.newFixedThreadPool(allowMultithreading ? Math.max(Runtime.getRuntime().availableProcessors(), 2) : 1);
+
+        prefetchIterator();
+    }
+
+    @Override
+    public void reset() {
+        this.executorService.shutdownNow();
+        this.iterator.reset();
+        underlyingHas.set(true);
+        prefetchIterator();
+        this.buffer.clear();
+        this.executorService = Executors.newFixedThreadPool(allowMultithreading ? Math.max(Runtime.getRuntime().availableProcessors(), 2) : 1);
+    }
+
+    public void shutdown() {
+        executorService.shutdown();
+    }
+
+    private static class CallableTransformer implements Callable<Sequence<VocabWord>> {
+
+        private LabelledDocument document;
+        private SentenceTransformer transformer;
+
+        public CallableTransformer(LabelledDocument document, SentenceTransformer transformer) {
+            this.transformer = transformer;
+            this.document = document;
+        }
+
+        @Override
+        public Sequence<VocabWord> call() {
+            Sequence<VocabWord> sequence = new Sequence<>();
+
+            if (document != null && document.getContent() != null) {
+                sequence = transformer.transformToSequence(document.getContent());
+                if (document.getLabels() != null) {
+                    for (String label : document.getLabels()) {
+                        if (label != null && !label.isEmpty())
+                            sequence.addSequenceLabel(new VocabWord(1.0, label));
+                    }
+                }
+            }
+            return sequence;
+        }
+    }
+
+    @Override
+    public boolean hasNext() {
+        //boolean before = underlyingHas;
+
+        //if (underlyingHas.get()) {
+            if (buffer.size() < capacity && iterator.hasNextDocument()) {
+                CallableTransformer transformer = new CallableTransformer(iterator.nextDocument(), sentenceTransformer);
+                Future<Sequence<VocabWord>> futureSequence = executorService.submit(transformer);
+                try {
+                    buffer.put(futureSequence);
+                } catch (InterruptedException e) {
+                    log.error("",e);
+                }
+            }
+          /*  else
+                underlyingHas.set(false);
+
+        }
+        else {
+           underlyingHas.set(false);
+        }*/
+
+        return (/*underlyingHas.get() ||*/ !buffer.isEmpty() || /*!stringBuffer.isEmpty() ||*/ processing.get() > 0);
+    }
+
+    @Override
+    public Sequence<VocabWord> next() {
+        try {
+            /*if (underlyingHas)
+                stringBuffer.put(iterator.nextDocument());*/
+            processing.incrementAndGet();
+            Future<Sequence<VocabWord>> future = buffer.take();
+            Sequence<VocabWord>  sequence = future.get();
+            processing.decrementAndGet();
+            return sequence;
+
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/Huffman.java

@@ -0,0 +1,184 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.word2vec;
+
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+
+/**
+ * Huffman tree builder
+ * @author Adam Gibson
+ *
+ */
+public class Huffman {
+
+    public final int MAX_CODE_LENGTH;
+    private volatile boolean buildTrigger = false;
+
+    private Logger logger = LoggerFactory.getLogger(Huffman.class);
+
+    public Huffman(Collection<? extends SequenceElement> words) {
+        this(words, 40);
+    }
+
+    /**
+     * Builds Huffman tree for collection of SequenceElements, with defined CODE_LENGTH
+     * Default CODE_LENGTH is 40
+     *
+     * @param words
+     * @param CODE_LENGTH CODE_LENGTH defines maximum length of code path, and effectively limits vocabulary size.
+     */
+    public Huffman(Collection<? extends SequenceElement> words, int CODE_LENGTH) {
+        this.MAX_CODE_LENGTH = CODE_LENGTH;
+        this.words = new ArrayList<>(words);
+        Collections.sort(this.words, new Comparator<SequenceElement>() {
+            @Override
+            public int compare(SequenceElement o1, SequenceElement o2) {
+                return Double.compare(o2.getElementFrequency(), o1.getElementFrequency());
+            }
+
+        });
+    }
+
+    private List<? extends SequenceElement> words;
+
+    public void build() {
+        buildTrigger = true;
+        long[] count = new long[words.size() * 2 + 1];
+        byte[] binary = new byte[words.size() * 2 + 1];
+        byte[] code = new byte[MAX_CODE_LENGTH];
+        int[] point = new int[MAX_CODE_LENGTH];
+        int[] parentNode = new int[words.size() * 2 + 1];
+        int a = 0;
+
+        while (a < words.size()) {
+            count[a] = (long) words.get(a).getElementFrequency();
+            a++;
+        }
+
+        a = words.size();
+
+        while (a < words.size() * 2) {
+            count[a] = Integer.MAX_VALUE;
+            a++;
+        }
+
+        int pos1 = words.size() - 1;
+        int pos2 = words.size();
+
+        int min1i;
+        int min2i;
+
+        a = 0;
+        // Following algorithm constructs the Huffman tree by adding one node at a time
+        for (a = 0; a < words.size() - 1; a++) {
+            // First, find two smallest nodes 'min1, min2'
+            if (pos1 >= 0) {
+                if (count[pos1] < count[pos2]) {
+                    min1i = pos1;
+                    pos1--;
+                } else {
+                    min1i = pos2;
+                    pos2++;
+                }
+            } else {
+                min1i = pos2;
+                pos2++;
+            }
+            if (pos1 >= 0) {
+                if (count[pos1] < count[pos2]) {
+                    min2i = pos1;
+                    pos1--;
+                } else {
+                    min2i = pos2;
+                    pos2++;
+                }
+            } else {
+                min2i = pos2;
+                pos2++;
+            }
+
+            count[words.size() + a] = count[min1i] + count[min2i];
+            parentNode[min1i] = words.size() + a;
+            parentNode[min2i] = words.size() + a;
+            binary[min2i] = 1;
+        }
+        // Now assign binary code to each vocabulary word
+        int i;
+        int b;
+        // Now assign binary code to each vocabulary word
+        for (a = 0; a < words.size(); a++) {
+            b = a;
+            i = 0;
+            do {
+                code[i] = binary[b];
+                point[i] = b;
+                i++;
+                b = parentNode[b];
+
+            } while (b != words.size() * 2 - 2 && i < 39);
+
+
+            words.get(a).setCodeLength((short) i);
+            words.get(a).getPoints().add(words.size() - 2);
+
+            for (b = 0; b < i; b++) {
+                try {
+                    words.get(a).getCodes().set(i - b - 1, code[b]);
+                    words.get(a).getPoints().set(i - b, point[b] - words.size());
+                } catch (Exception e) {
+                    logger.info("Words size: [" + words.size() + "], a: [" + a + "], b: [" + b + "], i: [" + i
+                                    + "], points size: [" + words.get(a).getPoints().size() + "]");
+                    throw new RuntimeException(e);
+                }
+            }
+
+        }
+
+
+    }
+
+    /**
+     * This method updates VocabCache and all it's elements with Huffman indexes
+     * Please note: it should be the same VocabCache as was used for Huffman tree initialization
+     *
+     * @param cache VocabCache to be updated.
+     */
+    public void applyIndexes(VocabCache<? extends SequenceElement> cache) {
+        if (!buildTrigger)
+            build();
+
+        for (int a = 0; a < words.size(); a++) {
+            if (words.get(a).getLabel() != null) {
+                cache.addWordToIndex(a, words.get(a).getLabel());
+            } else {
+                cache.addWordToIndex(a, words.get(a).getStorageId());
+            }
+
+            words.get(a).setIndex(a);
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/InputStreamCreator.java

@@ -0,0 +1,34 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.word2vec;
+
+import java.io.InputStream;
+import java.io.Serializable;
+
+public interface InputStreamCreator extends Serializable {
+
+
+    /**
+     * Create an input stream
+     * @return
+     */
+    InputStream create();
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/StaticWord2Vec.java

@@ -0,0 +1,427 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.word2vec;
+
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.embeddings.reader.ModelUtils;
+import org.deeplearning4j.models.embeddings.wordvectors.WordVectors;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.compression.AbstractStorage;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.ops.transforms.Transforms;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+@Slf4j
+public class StaticWord2Vec implements WordVectors {
+    private List<Map<Integer, INDArray>> cacheWrtDevice = new ArrayList<>();
+    private AbstractStorage<Integer> storage;
+    private long cachePerDevice = 0L;
+    private VocabCache<VocabWord> vocabCache;
+    private String unk = null;
+
+    private StaticWord2Vec() {
+
+    }
+
+    @Override
+    public String getUNK() {
+        return unk;
+    }
+
+    @Override
+    public void setUNK(String newUNK) {
+        this.unk = newUNK;
+    }
+
+    /**
+     * Init method validates configuration defined using
+     */
+    protected void init() {
+        if (storage.size() != vocabCache.numWords())
+            throw new RuntimeException("Number of words in Vocab isn't matching number of stored Vectors. vocab: ["
+                            + vocabCache.numWords() + "]; storage: [" + storage.size() + "]");
+
+        // initializing device cache
+        for (int i = 0; i < Nd4j.getAffinityManager().getNumberOfDevices(); i++) {
+            cacheWrtDevice.add(new ConcurrentHashMap<Integer, INDArray>());
+        }
+    }
+
+    /**
+     * Returns true if the model has this word in the vocab
+     *
+     * @param word the word to test for
+     * @return true if the model has the word in the vocab
+     */
+    @Override
+    public boolean hasWord(String word) {
+        return vocabCache.containsWord(word);
+    }
+
+    @Override
+    public Collection<String> wordsNearest(INDArray words, int top) {
+        throw new UnsupportedOperationException("Method isn't implemented. Please use usual Word2Vec implementation");
+    }
+
+    @Override
+    public Collection<String> wordsNearestSum(INDArray words, int top) {
+        throw new UnsupportedOperationException("Method isn't implemented. Please use usual Word2Vec implementation");
+    }
+
+    /**
+     * Get the top n words most similar to the given word
+     * PLEASE NOTE: This method is not available in this implementation.
+     *
+     * @param word the word to compare
+     * @param n    the n to get
+     * @return the top n words
+     */
+    @Override
+    public Collection<String> wordsNearestSum(String word, int n) {
+        throw new UnsupportedOperationException("Method isn't implemented. Please use usual Word2Vec implementation");
+    }
+
+    /**
+     * Words nearest based on positive and negative words
+     * PLEASE NOTE: This method is not available in this implementation.
+     *
+     * @param positive the positive words
+     * @param negative the negative words
+     * @param top      the top n words
+     * @return the words nearest the mean of the words
+     */
+    @Override
+    public Collection<String> wordsNearestSum(Collection<String> positive, Collection<String> negative, int top) {
+        throw new UnsupportedOperationException("Method isn't implemented. Please use usual Word2Vec implementation");
+    }
+
+    /**
+     * Accuracy based on questions which are a space separated list of strings
+     * where the first word is the query word, the next 2 words are negative,
+     * and the last word is the predicted word to be nearest
+     * PLEASE NOTE: This method is not available in this implementation.
+     *
+     * @param questions the questions to ask
+     * @return the accuracy based on these questions
+     */
+    @Override
+    public Map<String, Double> accuracy(List<String> questions) {
+        throw new UnsupportedOperationException("Method isn't implemented. Please use usual Word2Vec implementation");
+    }
+
+    @Override
+    public int indexOf(String word) {
+        return vocabCache.indexOf(word);
+    }
+
+    /**
+     * Find all words with a similar characters
+     * in the vocab
+     * PLEASE NOTE: This method is not available in this implementation.
+     *
+     * @param word     the word to compare
+     * @param accuracy the accuracy: 0 to 1
+     * @return the list of words that are similar in the vocab
+     */
+    @Override
+    public List<String> similarWordsInVocabTo(String word, double accuracy) {
+        throw new UnsupportedOperationException("Method isn't implemented. Please use usual Word2Vec implementation");
+    }
+
+    /**
+     * Get the word vector for a given matrix
+     *
+     * @param word the word to get the matrix for
+     * @return the ndarray for this word
+     */
+    @Override
+    public double[] getWordVector(String word) {
+        return getWordVectorMatrix(word).data().asDouble();
+    }
+
+    /**
+     * Returns the word vector divided by the norm2 of the array
+     *
+     * @param word the word to get the matrix for
+     * @return the looked up matrix
+     */
+    @Override
+    public INDArray getWordVectorMatrixNormalized(String word) {
+        return Transforms.unitVec(getWordVectorMatrix(word));
+    }
+
+    /**
+     * Get the word vector for a given matrix
+     *
+     * @param word the word to get the matrix for
+     * @return the ndarray for this word
+     */
+    @Override
+    public INDArray getWordVectorMatrix(String word) {
+        // TODO: add variable UNK here
+        int idx = 0;
+        if (hasWord(word))
+            idx = vocabCache.indexOf(word);
+        else if (getUNK() != null)
+            idx = vocabCache.indexOf(getUNK());
+        else
+            return null;
+
+        int deviceId = Nd4j.getAffinityManager().getDeviceForCurrentThread();
+        INDArray array = null;
+
+        if (cachePerDevice > 0 && cacheWrtDevice.get(deviceId).containsKey(idx))
+            return cacheWrtDevice.get(Nd4j.getAffinityManager().getDeviceForCurrentThread()).get(idx);
+
+        array = storage.get(idx);
+
+        if (cachePerDevice > 0) {
+            // TODO: add cache here
+            long arrayBytes = array.length() * array.data().getElementSize();
+            if ((arrayBytes * cacheWrtDevice.get(deviceId).size()) + arrayBytes < cachePerDevice)
+                cacheWrtDevice.get(deviceId).put(idx, array);
+        }
+
+        return array;
+    }
+
+    /**
+     * This method returns 2D array, where each row represents corresponding word/label
+     *
+     * @param labels
+     * @return
+     */
+    @Override
+    public INDArray getWordVectors(Collection<String> labels) {
+        List<INDArray> words = new ArrayList<>();
+        for (String label : labels) {
+            if (hasWord(label) || getUNK() != null)
+                words.add(getWordVectorMatrix(label));
+        }
+
+        return Nd4j.vstack(words);
+    }
+
+    /**
+     * This method returns mean vector, built from words/labels passed in
+     *
+     * @param labels
+     * @return
+     */
+    @Override
+    public INDArray getWordVectorsMean(Collection<String> labels) {
+        INDArray matrix = getWordVectors(labels);
+
+        // TODO: check this (1)
+        return matrix.mean(1);
+    }
+
+    /**
+     * Words nearest based on positive and negative words
+     * PLEASE NOTE: This method is not available in this implementation.
+     *
+     * @param positive the positive words
+     * @param negative the negative words
+     * @param top      the top n words
+     * @return the words nearest the mean of the words
+     */
+    @Override
+    public Collection<String> wordsNearest(Collection<String> positive, Collection<String> negative, int top) {
+        throw new UnsupportedOperationException("Method isn't implemented. Please use usual Word2Vec implementation");
+    }
+
+    /**
+     * Get the top n words most similar to the given word
+     * PLEASE NOTE: This method is not available in this implementation.
+     *
+     * @param word the word to compare
+     * @param n    the n to get
+     * @return the top n words
+     */
+    @Override
+    public Collection<String> wordsNearest(String word, int n) {
+        throw new UnsupportedOperationException("Method isn't implemented. Please use usual Word2Vec implementation");
+    }
+
+    /**
+     * Returns the similarity of 2 words
+     *
+     * @param label1  the first word
+     * @param label2 the second word
+     * @return a normalized similarity (cosine similarity)
+     */
+    @Override
+    public double similarity(String label1, String label2) {
+        if (label1 == null || label2 == null) {
+            log.debug("LABELS: " + label1 + ": " + (label1 == null ? "null" : "exists") + ";" + label2 + " vec2:"
+                            + (label2 == null ? "null" : "exists"));
+            return Double.NaN;
+        }
+
+        INDArray vec1 = getWordVectorMatrix(label1).dup();
+        INDArray vec2 = getWordVectorMatrix(label2).dup();
+
+        if (vec1 == null || vec2 == null) {
+            log.debug(label1 + ": " + (vec1 == null ? "null" : "exists") + ";" + label2 + " vec2:"
+                            + (vec2 == null ? "null" : "exists"));
+            return Double.NaN;
+        }
+
+        if (label1.equals(label2))
+            return 1.0;
+
+        vec1 = Transforms.unitVec(vec1);
+        vec2 = Transforms.unitVec(vec2);
+
+        return Transforms.cosineSim(vec1, vec2);
+    }
+
+    /**
+     * Vocab for the vectors
+     *
+     * @return
+     */
+    @Override
+    public VocabCache vocab() {
+        return vocabCache;
+    }
+
+    /**
+     * Lookup table for the vectors
+     * PLEASE NOTE: This method is not available in this implementation.
+     *
+     * @return
+     */
+    @Override
+    public WeightLookupTable lookupTable() {
+        throw new UnsupportedOperationException("Method isn't implemented. Please use usual Word2Vec implementation");
+    }
+
+    /**
+     * Specifies ModelUtils to be used to access model
+     * PLEASE NOTE: This method has no effect in this implementation.
+     *
+     * @param utils
+     */
+    @Override
+    public void setModelUtils(ModelUtils utils) {
+        // no-op
+    }
+
+    @Override
+    public void loadWeightsInto(INDArray array) {
+        int n = (int)vocabSize();
+        INDArray zero = null;
+        for( int i=0; i<n; i++ ){
+            INDArray arr = storage.get(i);
+            if(arr == null){    //TODO is this even possible?
+                if(zero == null)
+                    zero = Nd4j.create(array.dataType(), 1, array.size(1));
+                arr = zero;
+            }
+            array.putRow(i, arr);
+        }
+    }
+
+    @Override
+    public long vocabSize() {
+        return storage.size();
+    }
+
+    @Override
+    public int vectorSize() {
+        INDArray arr = storage.get(0);
+        if(arr != null)
+            return (int)arr.length();
+
+        int vs = (int)vocabSize();
+        for( int i=1; i<vs; i++ ){
+            arr = storage.get(0);
+            if(arr != null)
+                return (int)arr.length();
+        }
+        throw new UnsupportedOperationException("No vectors found");
+    }
+
+    @Override
+    public boolean jsonSerializable() {
+        return false;
+    }
+
+    @Override
+    public boolean outOfVocabularySupported() {
+        return false;
+    }
+
+    public static class Builder {
+
+        private AbstractStorage<Integer> storage;
+        private long cachePerDevice = 0L;
+        private VocabCache<VocabWord> vocabCache;
+
+        /**
+         *
+         * @param storage AbstractStorage implementation, key has to be Integer, index of vocabWords
+         * @param vocabCache VocabCache implementation, which will be used to lookup word indexes
+         */
+        public Builder(AbstractStorage<Integer> storage, VocabCache<VocabWord> vocabCache) {
+            this.storage = storage;
+            this.vocabCache = vocabCache;
+        }
+
+
+        /**
+         * This method lets you to define if decompressed values will be cached, to avoid excessive decompressions.
+         * If bytes == 0 - no cache will be used.
+         *
+         * @param bytes
+         * @return
+         */
+        public Builder setCachePerDevice(long bytes) {
+            this.cachePerDevice = bytes;
+            return this;
+        }
+
+
+        /**
+         * This method returns Static Word2Vec implementation, which is suitable for tasks like neural nets feeding.
+         *
+         * @return
+         */
+        public StaticWord2Vec build() {
+            StaticWord2Vec word2Vec = new StaticWord2Vec();
+            word2Vec.cachePerDevice = this.cachePerDevice;
+            word2Vec.storage = this.storage;
+            word2Vec.vocabCache = this.vocabCache;
+
+            word2Vec.init();
+
+            return word2Vec;
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/StreamWork.java

@@ -0,0 +1,57 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.word2vec;
+
+import java.io.InputStream;
+import java.io.Serializable;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class StreamWork implements Serializable {
+    private InputStreamCreator is;
+    private AtomicInteger count = new AtomicInteger(0);
+
+
+    public StreamWork(InputStreamCreator is, AtomicInteger count) {
+        super();
+        this.is = is;
+        this.count = count;
+    }
+
+    public InputStream getIs() {
+        return is.create();
+    }
+
+    public AtomicInteger getCount() {
+        return count;
+    }
+
+    public void setCount(AtomicInteger count) {
+        this.count = count;
+    }
+
+    public void countDown() {
+        count.decrementAndGet();
+
+    }
+
+
+
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/VocabWord.java

@@ -0,0 +1,146 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.word2vec;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.nd4j.shade.jackson.annotation.JsonAutoDetect;
+import org.nd4j.shade.jackson.annotation.JsonTypeInfo;
+import org.nd4j.shade.jackson.databind.ObjectMapper;
+
+import java.io.Serializable;
+
+
+/**
+ * Intermediate layers of the neural network
+ *
+ * @author Adam Gibson
+ */
+@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class", defaultImpl =  VocabWord.class)
+@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE,
+        setterVisibility = JsonAutoDetect.Visibility.NONE)
+public class VocabWord extends SequenceElement implements Serializable {
+
+    private static final long serialVersionUID = 2223750736522624256L;
+
+    //for my sanity
+    private String word;
+
+    /*
+        Used for Joint/Distributed vocabs mechanics
+     */
+    @Getter
+    @Setter
+    protected long vocabId;
+    @Getter
+    @Setter
+    protected long affinityId;
+
+    public static VocabWord none() {
+        return new VocabWord(0, "none");
+    }
+
+    /**
+     *
+     * @param wordFrequency count of the word
+    
+     */
+    public VocabWord(double wordFrequency, @NonNull String word) {
+        if (word.isEmpty())
+            throw new IllegalArgumentException("Word must not be null or empty");
+        this.word = word;
+        this.elementFrequency.set(wordFrequency);
+        this.storageId = SequenceElement.getLongHash(word);
+    }
+
+    public VocabWord(double wordFrequency, @NonNull String word, long storageId) {
+        this(wordFrequency, word);
+        this.storageId = storageId;
+    }
+
+
+    public VocabWord() {}
+
+
+    public String getLabel() {
+        return this.word;
+    }
+
+
+    public String getWord() {
+        return word;
+    }
+
+    public void setWord(String word) {
+        this.word = word;
+    }
+
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (!(o instanceof VocabWord))
+            return false;
+        final VocabWord vocabWord = (VocabWord) o;
+        if (this.word == null)
+            return vocabWord.word == null;
+
+        return this.word.equals(vocabWord.getWord());
+    }
+    
+
+    @Override
+    public int hashCode() {
+        final int result = this.word == null ? 0 : this.word.hashCode(); //this.elementFrequency.hashCode();
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "VocabWord{" +
+                "word='" + word + '\'' +
+                ", elementFrequency=" + elementFrequency +
+                ", index=" + index +
+                ", codes=" + codes +
+                ", points=" + points +
+                ", codeLength=" + codeLength +
+                '}';
+    }
+
+    @Override
+    public String toJSON() {
+        ObjectMapper mapper = mapper();
+        try {
+            /*
+                we need JSON as single line to save it at first line of the CSV model file
+            */
+            return mapper.writeValueAsString(this);
+        } catch (org.nd4j.shade.jackson.core.JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
+}
+

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/VocabWork.java

@@ -0,0 +1,116 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.word2vec;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class VocabWork implements Serializable {
+
+    private AtomicInteger count = new AtomicInteger(0);
+    private String work;
+    private boolean stem = false;
+    private List<String> label;
+
+
+    public VocabWork(AtomicInteger count, String work, boolean stem) {
+        this(count, work, stem, "");
+    }
+
+
+
+    public VocabWork(AtomicInteger count, String work, boolean stem, String label) {
+        this(count, work, stem, Arrays.asList(label));
+    }
+
+    public VocabWork(AtomicInteger count, String work, boolean stem, List<String> label) {
+        this.count = count;
+        this.work = work;
+        this.stem = stem;
+        this.label = label;
+    }
+
+    public AtomicInteger getCount() {
+        return count;
+    }
+
+    public void setCount(AtomicInteger count) {
+        this.count = count;
+    }
+
+    public String getWork() {
+        return work;
+    }
+
+    public void setWork(String work) {
+        this.work = work;
+    }
+
+    public void increment() {
+        count.incrementAndGet();
+    }
+
+    public boolean isStem() {
+        return stem;
+    }
+
+    public void setStem(boolean stem) {
+        this.stem = stem;
+    }
+
+    public List<String> getLabel() {
+        return label;
+    }
+
+    public void setLabel(List<String> label) {
+        this.label = label;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (!(o instanceof VocabWork))
+            return false;
+
+        VocabWork vocabWork = (VocabWork) o;
+
+        if (stem != vocabWork.stem)
+            return false;
+        if (count != null ? !count.equals(vocabWork.count) : vocabWork.count != null)
+            return false;
+        if (label != null ? !label.equals(vocabWork.label) : vocabWork.label != null)
+            return false;
+        return !(work != null ? !work.equals(vocabWork.work) : vocabWork.work != null);
+
+    }
+
+    @Override
+    public int hashCode() {
+        int result = count != null ? count.hashCode() : 0;
+        result = 31 * result + (work != null ? work.hashCode() : 0);
+        result = 31 * result + (stem ? 1 : 0);
+        result = 31 * result + (label != null ? label.hashCode() : 0);
+        return result;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/Word2Vec.java

@@ -0,0 +1,790 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.word2vec;
+
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import lombok.Getter;
+import lombok.NonNull;
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.embeddings.learning.ElementsLearningAlgorithm;
+import org.deeplearning4j.models.embeddings.loader.VectorsConfiguration;
+import org.deeplearning4j.models.embeddings.reader.ModelUtils;
+import org.deeplearning4j.models.embeddings.wordvectors.WordVectors;
+import org.deeplearning4j.models.paragraphvectors.ParagraphVectors;
+import org.deeplearning4j.models.sequencevectors.SequenceVectors;
+import org.deeplearning4j.models.sequencevectors.interfaces.SequenceIterator;
+import org.deeplearning4j.models.sequencevectors.interfaces.VectorsListener;
+import org.deeplearning4j.models.sequencevectors.iterators.AbstractSequenceIterator;
+import org.deeplearning4j.models.sequencevectors.transformers.impl.SentenceTransformer;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.deeplearning4j.models.word2vec.wordstore.inmemory.AbstractCache;
+import org.deeplearning4j.text.documentiterator.DocumentIterator;
+import org.deeplearning4j.text.documentiterator.LabelAwareIterator;
+import org.deeplearning4j.text.sentenceiterator.SentenceIterator;
+import org.deeplearning4j.text.sentenceiterator.StreamLineIterator;
+import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory;
+import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
+import org.nd4j.shade.jackson.core.JsonProcessingException;
+import org.nd4j.shade.jackson.databind.DeserializationFeature;
+import org.nd4j.shade.jackson.databind.ObjectMapper;
+import org.nd4j.shade.jackson.databind.SerializationFeature;
+import org.nd4j.shade.jackson.databind.type.CollectionType;
+
+import java.io.IOException;
+import java.util.*;
+
+public class Word2Vec extends SequenceVectors<VocabWord> {
+    private static final long serialVersionUID = 78249242142L;
+
+    protected transient SentenceIterator sentenceIter;
+    @Getter
+    protected transient TokenizerFactory tokenizerFactory;
+
+    /**
+     * This method defines TokenizerFactory instance to be using during model building
+     *
+     * @param tokenizerFactory TokenizerFactory instance
+     */
+    public void setTokenizerFactory(@NonNull TokenizerFactory tokenizerFactory) {
+        this.tokenizerFactory = tokenizerFactory;
+
+        if (sentenceIter != null) {
+            SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(sentenceIter)
+                    .vocabCache(vocab)
+                    .tokenizerFactory(this.tokenizerFactory).build();
+            this.iterator = new AbstractSequenceIterator.Builder<>(transformer).build();
+        }
+    }
+
+    /**
+     * This method defines SentenceIterator instance, that will be used as training corpus source
+     *
+     * @param iterator SentenceIterator instance
+     */
+    public void setSentenceIterator(@NonNull SentenceIterator iterator) {
+
+        if (tokenizerFactory != null) {
+            SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(iterator)
+                    .tokenizerFactory(tokenizerFactory)
+                    .vocabCache(vocab)
+                    .allowMultithreading(configuration == null || configuration.isAllowParallelTokenization())
+                    .build();
+            this.iterator = new AbstractSequenceIterator.Builder<>(transformer).build();
+        } else
+            log.error("Please call setTokenizerFactory() prior to setSentenceIter() call.");
+    }
+
+    /**
+     * This method defines SequenceIterator instance, that will be used as training corpus source.
+     * Main difference with other iterators here: it allows you to pass already tokenized Sequence<VocabWord> for training
+     *
+     * @param iterator
+     */
+    public void setSequenceIterator(@NonNull SequenceIterator<VocabWord> iterator) {
+        this.iterator = iterator;
+    }
+
+    private static ObjectMapper mapper = null;
+    private static final Object lock = new Object();
+
+    private static ObjectMapper mapper() {
+        if (mapper == null) {
+            synchronized (lock) {
+                if (mapper == null) {
+                    mapper = new ObjectMapper();
+                    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+                    mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
+                    return mapper;
+                }
+            }
+        }
+        return mapper;
+    }
+
+    private static final String CLASS_FIELD = "@class";
+    private static final String VOCAB_LIST_FIELD = "VocabCache";
+
+    public String toJson() throws JsonProcessingException {
+
+        JsonObject retVal = new JsonObject();
+        ObjectMapper mapper = mapper();
+
+        retVal.addProperty(CLASS_FIELD, mapper.writeValueAsString(this.getClass().getName()));
+
+        if (this.vocab instanceof AbstractCache) {
+            retVal.addProperty(VOCAB_LIST_FIELD, ((AbstractCache<VocabWord>) this.vocab).toJson());
+        }
+
+        return retVal.toString();
+    }
+
+    public static Word2Vec fromJson(String jsonString)  throws IOException {
+
+        Word2Vec ret = new Word2Vec();
+
+        JsonParser parser = new JsonParser();
+        JsonObject json = parser.parse(jsonString).getAsJsonObject();
+
+        VocabCache cache = AbstractCache.fromJson(json.get(VOCAB_LIST_FIELD).getAsString());
+
+        ret.setVocab(cache);
+        return ret;
+    }
+
+    @Override
+    public String toString() {
+        return "Word2Vec{" +
+                "sentenceIter=" + sentenceIter +
+                ", tokenizerFactory=" + tokenizerFactory +
+                ", iterator=" + iterator +
+                ", elementsLearningAlgorithm=" + elementsLearningAlgorithm +
+                ", sequenceLearningAlgorithm=" + sequenceLearningAlgorithm +
+                ", configuration=" + configuration +
+                ", existingModel=" + existingModel +
+                ", intersectModel=" + intersectModel +
+                ", unknownElement=" + unknownElement +
+                ", scoreElements=" + scoreElements +
+                ", scoreSequences=" + scoreSequences +
+                ", configured=" + configured +
+                ", lockFactor=" + lockFactor +
+                ", enableScavenger=" + enableScavenger +
+                ", vocabLimit=" + vocabLimit +
+                ", eventListeners=" + eventListeners +
+                ", minWordFrequency=" + minWordFrequency +
+                ", lookupTable=" + lookupTable +
+                ", vocab=" + vocab +
+                ", layerSize=" + layerSize +
+                ", modelUtils=" + modelUtils +
+                ", numIterations=" + numIterations +
+                ", numEpochs=" + numEpochs +
+                ", negative=" + negative +
+                ", sampling=" + sampling +
+                ", learningRate=" + learningRate +
+                ", minLearningRate=" + minLearningRate +
+                ", window=" + window +
+                ", batchSize=" + batchSize +
+                ", learningRateDecayWords=" + learningRateDecayWords +
+                ", resetModel=" + resetModel +
+                ", useAdeGrad=" + useAdeGrad +
+                ", workers=" + workers +
+                ", trainSequenceVectors=" + trainSequenceVectors +
+                ", trainElementsVectors=" + trainElementsVectors +
+                ", seed=" + seed +
+                ", useUnknown=" + useUnknown +
+                ", variableWindows=" + Arrays.toString(variableWindows) +
+                ", stopWords=" + stopWords +
+                '}';
+    }
+
+    public static class Builder extends SequenceVectors.Builder<VocabWord> {
+        protected SentenceIterator sentenceIterator;
+        protected LabelAwareIterator labelAwareIterator;
+        protected TokenizerFactory tokenizerFactory;
+        protected boolean allowParallelTokenization = true;
+
+
+        public Builder() {
+
+        }
+
+        /**
+         * This method has no effect for Word2Vec
+         *
+         * @param vec existing WordVectors model
+         * @return
+         */
+        @Override
+        protected Builder useExistingWordVectors(@NonNull WordVectors vec) {
+            return this;
+        }
+
+        public Builder(@NonNull VectorsConfiguration configuration) {
+            super(configuration);
+            this.allowParallelTokenization = configuration.isAllowParallelTokenization();
+        }
+
+        public Builder iterate(@NonNull DocumentIterator iterator) {
+            this.sentenceIterator = new StreamLineIterator.Builder(iterator).setFetchSize(100).build();
+            return this;
+        }
+
+        /**
+         * This method used to feed SentenceIterator, that contains training corpus, into ParagraphVectors
+         *
+         * @param iterator
+         * @return
+         */
+        public Builder iterate(@NonNull SentenceIterator iterator) {
+            this.sentenceIterator = iterator;
+            return this;
+        }
+
+        /**
+         * This method defines TokenizerFactory to be used for strings tokenization during training
+         * PLEASE NOTE: If external VocabCache is used, the same TokenizerFactory should be used to keep derived tokens equal.
+         *
+         * @param tokenizerFactory
+         * @return
+         */
+        public Builder tokenizerFactory(@NonNull TokenizerFactory tokenizerFactory) {
+            this.tokenizerFactory = tokenizerFactory;
+            return this;
+        }
+
+        /**
+         * This method used to feed SequenceIterator, that contains training corpus, into ParagraphVectors
+         *
+         * @param iterator
+         * @return
+         */
+        @Override
+        public Builder iterate(@NonNull SequenceIterator<VocabWord> iterator) {
+            super.iterate(iterator);
+            return this;
+        }
+
+        /**
+         * This method used to feed LabelAwareIterator, that is usually used
+         *
+         * @param iterator
+         * @return
+         */
+        public Builder iterate(@NonNull LabelAwareIterator iterator) {
+            this.labelAwareIterator = iterator;
+            return this;
+        }
+
+        /**
+         * This method defines mini-batch size
+         * @param batchSize
+         * @return
+         */
+        @Override
+        public Builder batchSize(int batchSize) {
+            super.batchSize(batchSize);
+            return this;
+        }
+
+        /**
+         * This method defines number of iterations done for each mini-batch during training
+         * @param iterations
+         * @return
+         */
+        @Override
+        public Builder iterations(int iterations) {
+            super.iterations(iterations);
+            return this;
+        }
+
+        /**
+         * This method defines number of epochs (iterations over whole training corpus) for training
+         * @param numEpochs
+         * @return
+         */
+        @Override
+        public Builder epochs(int numEpochs) {
+            super.epochs(numEpochs);
+            return this;
+        }
+
+        /**
+         * This method defines number of dimensions for output vectors
+         * @param layerSize
+         * @return
+         */
+        @Override
+        public Builder layerSize(int layerSize) {
+            super.layerSize(layerSize);
+            return this;
+        }
+
+        /**
+         * This method defines initial learning rate for model training
+         *
+         * @param learningRate
+         * @return
+         */
+        @Override
+        public Builder learningRate(double learningRate) {
+            super.learningRate(learningRate);
+            return this;
+        }
+
+        /**
+         * This method defines minimal word frequency in training corpus. All words below this threshold will be removed prior model training
+         *
+         * @param minWordFrequency
+         * @return
+         */
+        @Override
+        public Builder minWordFrequency(int minWordFrequency) {
+            super.minWordFrequency(minWordFrequency);
+            return this;
+        }
+
+        /**
+         * This method defines minimal learning rate value for training
+         *
+         * @param minLearningRate
+         * @return
+         */
+        @Override
+        public Builder minLearningRate(double minLearningRate) {
+            super.minLearningRate(minLearningRate);
+            return this;
+        }
+
+        /**
+         * This method defines whether model should be totally wiped out prior building, or not
+         *
+         * @param reallyReset
+         * @return
+         */
+        @Override
+        public Builder resetModel(boolean reallyReset) {
+            super.resetModel(reallyReset);
+            return this;
+        }
+
+        /**
+         * This method sets vocabulary limit during construction.
+         *
+         * Default value: 0. Means no limit
+         *
+         * @param limit
+         * @return
+         */
+        @Override
+        public Builder limitVocabularySize(int limit) {
+            super.limitVocabularySize(limit);
+            return this;
+        }
+
+        /**
+         * This method allows to define external VocabCache to be used
+         *
+         * @param vocabCache
+         * @return
+         */
+        @Override
+        public Builder vocabCache(@NonNull VocabCache<VocabWord> vocabCache) {
+            super.vocabCache(vocabCache);
+            return this;
+        }
+
+        /**
+         * This method allows to define external WeightLookupTable to be used
+         *
+         * @param lookupTable
+         * @return
+         */
+        @Override
+        public Builder lookupTable(@NonNull WeightLookupTable<VocabWord> lookupTable) {
+            super.lookupTable(lookupTable);
+            return this;
+        }
+
+        /**
+         * This method defines whether subsampling should be used or not
+         *
+         * @param sampling set > 0 to subsampling argument, or 0 to disable
+         * @return
+         */
+        @Override
+        public Builder sampling(double sampling) {
+            super.sampling(sampling);
+            return this;
+        }
+
+        /**
+         * This method defines whether adaptive gradients should be used or not
+         *
+         * @param reallyUse
+         * @return
+         */
+        @Override
+        public Builder useAdaGrad(boolean reallyUse) {
+            super.useAdaGrad(reallyUse);
+            return this;
+        }
+
+        /**
+         * This method defines whether negative sampling should be used or not
+         *
+         * PLEASE NOTE: If you're going to use negative sampling, you might want to disable HierarchicSoftmax, which is enabled by default
+         *
+         * Default value: 0
+         *
+         * @param negative set > 0 as negative sampling argument, or 0 to disable
+         * @return
+         */
+        @Override
+        public Builder negativeSample(double negative) {
+            super.negativeSample(negative);
+            return this;
+        }
+
+        /**
+         * This method defines stop words that should be ignored during training
+         *
+         * @param stopList
+         * @return
+         */
+        @Override
+        public Builder stopWords(@NonNull List<String> stopList) {
+            super.stopWords(stopList);
+            return this;
+        }
+
+        /**
+         * This method is hardcoded to TRUE, since that's whole point of Word2Vec
+         *
+         * @param trainElements
+         * @return
+         */
+        @Override
+        public Builder trainElementsRepresentation(boolean trainElements) {
+            throw new IllegalStateException("You can't change this option for Word2Vec");
+        }
+
+        /**
+         * This method is hardcoded to FALSE, since that's whole point of Word2Vec
+         *
+         * @param trainSequences
+         * @return
+         */
+        @Override
+        public Builder trainSequencesRepresentation(boolean trainSequences) {
+            throw new IllegalStateException("You can't change this option for Word2Vec");
+        }
+
+        /**
+         * This method defines stop words that should be ignored during training
+         *
+         * @param stopList
+         * @return
+         */
+        @Override
+        public Builder stopWords(@NonNull Collection<VocabWord> stopList) {
+            super.stopWords(stopList);
+            return this;
+        }
+
+        /**
+         * This method defines context window size
+         *
+         * @param windowSize
+         * @return
+         */
+        @Override
+        public Builder windowSize(int windowSize) {
+            super.windowSize(windowSize);
+            return this;
+        }
+
+        /**
+         * This method defines random seed for random numbers generator
+         * @param randomSeed
+         * @return
+         */
+        @Override
+        public Builder seed(long randomSeed) {
+            super.seed(randomSeed);
+            return this;
+        }
+
+
+        /**
+         * Sets number of threads running calculations.
+         * Note this is different from workers which affect
+         * the number of threads used to compute updates.
+         * This should be balanced with the number of workers.
+         * High number of threads will actually hinder performance.
+         *
+         * @param vectorCalcThreads the number of threads to compute updates
+         * @return
+         */
+        public Builder vectorCalcThreads(int vectorCalcThreads) {
+            super.vectorCalcThreads(vectorCalcThreads);
+            return this;
+        }
+
+
+        /**
+         * This method defines maximum number of concurrent threads available for training
+         *
+         * @param numWorkers
+         * @return
+         */
+        @Override
+        public Builder workers(int numWorkers) {
+            super.workers(numWorkers);
+            return this;
+        }
+
+        /**
+         * Sets ModelUtils that gonna be used as provider for utility methods: similarity(), wordsNearest(), accuracy(), etc
+         *
+         * @param modelUtils model utils to be used
+         * @return
+         */
+        @Override
+        public Builder modelUtils(@NonNull ModelUtils<VocabWord> modelUtils) {
+            super.modelUtils(modelUtils);
+            return this;
+        }
+
+        /**
+         * This method allows to use variable window size. In this case, every batch gets processed using one of predefined window sizes
+         *
+         * @param windows
+         * @return
+         */
+        @Override
+        public Builder useVariableWindow(int... windows) {
+            super.useVariableWindow(windows);
+            return this;
+        }
+
+        /**
+         * This method allows you to specify SequenceElement that will be used as UNK element, if UNK is used
+         *
+         * @param element
+         * @return
+         */
+        @Override
+        public Builder unknownElement(VocabWord element) {
+            super.unknownElement(element);
+            return this;
+        }
+
+        /**
+         * This method allows you to specify, if UNK word should be used internally
+         *
+         * @param reallyUse
+         * @return
+         */
+        @Override
+        public Builder useUnknown(boolean reallyUse) {
+            super.useUnknown(reallyUse);
+            if (this.unknownElement == null) {
+                this.unknownElement(new VocabWord(1.0, Word2Vec.DEFAULT_UNK));
+            }
+            return this;
+        }
+
+        /**
+         * This method sets VectorsListeners for this SequenceVectors model
+         *
+         * @param vectorsListeners
+         * @return
+         */
+        @Override
+        public Builder setVectorsListeners(@NonNull Collection<VectorsListener<VocabWord>> vectorsListeners) {
+            super.setVectorsListeners(vectorsListeners);
+            return this;
+        }
+
+        @Override
+        public Builder elementsLearningAlgorithm(String algorithm) {
+            if(algorithm == null)
+                return this;
+            super.elementsLearningAlgorithm(algorithm);
+            return this;
+        }
+
+        @Override
+        public Builder elementsLearningAlgorithm(ElementsLearningAlgorithm<VocabWord> algorithm) {
+          if(algorithm == null)
+              return this;
+            super.elementsLearningAlgorithm(algorithm);
+            return this;
+        }
+
+        /**
+         * This method enables/disables parallel tokenization.
+         *
+         * Default value: TRUE
+         * @param allow
+         * @return
+         */
+        public Builder allowParallelTokenization(boolean allow) {
+            this.allowParallelTokenization = allow;
+            return this;
+        }
+
+        /**
+         * This method ebables/disables periodical vocab truncation during construction
+         *
+         * Default value: disabled
+         *
+         * @param reallyEnable
+         * @return
+         */
+        @Override
+        public Builder enableScavenger(boolean reallyEnable) {
+            super.enableScavenger(reallyEnable);
+            return this;
+        }
+
+        /**
+         * This method enables/disables Hierarchic softmax
+         *
+         * Default value: enabled
+         *
+         * @param reallyUse
+         * @return
+         */
+        @Override
+        public Builder useHierarchicSoftmax(boolean reallyUse) {
+            super.useHierarchicSoftmax(reallyUse);
+            return this;
+        }
+
+        @Override
+        public Builder usePreciseWeightInit(boolean reallyUse) {
+            super.usePreciseWeightInit(reallyUse);
+            return this;
+        }
+
+        @Override
+        public Builder usePreciseMode(boolean reallyUse) {
+            super.usePreciseMode(reallyUse);
+            return this;
+        }
+
+        @Override
+        public Builder intersectModel(@NonNull SequenceVectors vectors, boolean isLocked) {
+            super.intersectModel(vectors, isLocked);
+            return this;
+        }
+
+        public Word2Vec build() {
+            presetTables();
+
+            Word2Vec ret = new Word2Vec();
+
+            if (sentenceIterator != null) {
+                if (tokenizerFactory == null)
+                    tokenizerFactory = new DefaultTokenizerFactory();
+
+                SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(sentenceIterator)
+                        .vocabCache(vocabCache)
+                        .tokenizerFactory(tokenizerFactory).allowMultithreading(allowParallelTokenization)
+                        .build();
+                this.iterator = new AbstractSequenceIterator.Builder<>(transformer).build();
+            }
+
+            if (this.labelAwareIterator != null) {
+                if (tokenizerFactory == null)
+                    tokenizerFactory = new DefaultTokenizerFactory();
+
+                SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(labelAwareIterator)
+                        .tokenizerFactory(tokenizerFactory).allowMultithreading(allowParallelTokenization)
+                        .vocabCache(vocabCache)
+                        .build();
+                this.iterator = new AbstractSequenceIterator.Builder<>(transformer).build();
+            }
+
+            ret.numEpochs = this.numEpochs;
+            ret.numIterations = this.iterations;
+            ret.vocab = this.vocabCache;
+            ret.minWordFrequency = this.minWordFrequency;
+            ret.learningRate.set(this.learningRate);
+            ret.minLearningRate = this.minLearningRate;
+            ret.sampling = this.sampling;
+            ret.negative = this.negative;
+            ret.layerSize = this.layerSize;
+            ret.batchSize = this.batchSize;
+            ret.learningRateDecayWords = this.learningRateDecayWords;
+            ret.window = this.window;
+            ret.resetModel = this.resetModel;
+            ret.useAdeGrad = this.useAdaGrad;
+            ret.stopWords = this.stopWords;
+            ret.workers = this.workers;
+            ret.useUnknown = this.useUnknown;
+            ret.unknownElement = this.unknownElement;
+            ret.variableWindows = this.variableWindows;
+            ret.seed = this.seed;
+            ret.enableScavenger = this.enableScavenger;
+            ret.vocabLimit = this.vocabLimit;
+
+            if (ret.unknownElement == null)
+                ret.unknownElement = new VocabWord(1.0,SequenceVectors.DEFAULT_UNK);
+
+
+            ret.iterator = this.iterator;
+            ret.lookupTable = this.lookupTable;
+            ret.tokenizerFactory = this.tokenizerFactory;
+            ret.modelUtils = this.modelUtils;
+
+            ret.elementsLearningAlgorithm = this.elementsLearningAlgorithm;
+            ret.sequenceLearningAlgorithm = this.sequenceLearningAlgorithm;
+
+            ret.intersectModel = this.intersectVectors;
+            ret.lockFactor = this.lockFactor;
+
+            if(!this.configurationSpecified) {
+                this.configuration.setLearningRate(this.learningRate);
+                this.configuration.setLayersSize(layerSize);
+                this.configuration.setHugeModelExpected(hugeModelExpected);
+                this.configuration.setWindow(window);
+                this.configuration.setMinWordFrequency(minWordFrequency);
+                this.configuration.setIterations(iterations);
+                this.configuration.setSeed(seed);
+                this.configuration.setBatchSize(batchSize);
+                this.configuration.setLearningRateDecayWords(learningRateDecayWords);
+                this.configuration.setMinLearningRate(minLearningRate);
+                this.configuration.setSampling(this.sampling);
+                this.configuration.setUseAdaGrad(useAdaGrad);
+                this.configuration.setNegative(negative);
+                this.configuration.setEpochs(this.numEpochs);
+                this.configuration.setStopList(this.stopWords);
+                this.configuration.setVariableWindows(variableWindows);
+                this.configuration.setUseHierarchicSoftmax(this.useHierarchicSoftmax);
+                this.configuration.setPreciseWeightInit(this.preciseWeightInit);
+                this.configuration.setModelUtils(this.modelUtils.getClass().getCanonicalName());
+                this.configuration.setAllowParallelTokenization(this.allowParallelTokenization);
+                this.configuration.setPreciseMode(this.preciseMode);
+
+            }
+
+            if (tokenizerFactory != null) {
+                this.configuration.setTokenizerFactory(tokenizerFactory.getClass().getCanonicalName());
+                if (tokenizerFactory.getTokenPreProcessor() != null)
+                    this.configuration.setTokenPreProcessor(
+                            tokenizerFactory.getTokenPreProcessor().getClass().getCanonicalName());
+            }
+
+            ret.configuration = this.configuration;
+
+            // we hardcode
+            ret.trainSequenceVectors = false;
+            ret.trainElementsVectors = true;
+
+            ret.eventListeners = this.vectorsListeners;
+
+
+            return ret;
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/iterator/Word2VecDataSetIterator.java

@@ -0,0 +1,288 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.word2vec.iterator;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.models.word2vec.Word2Vec;
+import org.deeplearning4j.text.inputsanitation.InputHomogenization;
+import org.deeplearning4j.text.movingwindow.Window;
+import org.deeplearning4j.text.movingwindow.WindowConverter;
+import org.deeplearning4j.text.movingwindow.Windows;
+import org.deeplearning4j.text.sentenceiterator.SentencePreProcessor;
+import org.deeplearning4j.text.sentenceiterator.labelaware.LabelAwareSentenceIterator;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.util.FeatureUtil;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+@Slf4j
+public class Word2VecDataSetIterator implements DataSetIterator {
+    private Word2Vec vec;
+    private LabelAwareSentenceIterator iter;
+    private List<Window> cachedWindow;
+    private List<String> labels;
+    private int batch = 10;
+    @Getter
+    private DataSetPreProcessor preProcessor;
+
+    /**
+     * Allows for customization of all of the params of the iterator
+     * @param vec the word2vec model to use
+     * @param iter the sentence iterator to use
+     * @param labels the possible labels
+     * @param batch the batch size
+     * @param homogenization whether to homogenize the sentences or not
+     * @param addLabels whether to add labels for windows
+     */
+    public Word2VecDataSetIterator(Word2Vec vec, LabelAwareSentenceIterator iter, List<String> labels, int batch,
+                    boolean homogenization, boolean addLabels) {
+        this.vec = vec;
+        this.iter = iter;
+        this.labels = labels;
+        this.batch = batch;
+        cachedWindow = new CopyOnWriteArrayList<>();
+
+        if (addLabels && homogenization)
+            iter.setPreProcessor(new SentencePreProcessor() {
+                @Override
+                public String preProcess(String sentence) {
+                    String label = Word2VecDataSetIterator.this.iter.currentLabel();
+                    String ret = "<" + label + "> " + new InputHomogenization(sentence).transform() + " </" + label
+                                    + ">";
+                    return ret;
+                }
+            });
+
+        else if (addLabels)
+            iter.setPreProcessor(new SentencePreProcessor() {
+                @Override
+                public String preProcess(String sentence) {
+                    String label = Word2VecDataSetIterator.this.iter.currentLabel();
+                    String ret = "<" + label + ">" + sentence + "</" + label + ">";
+                    return ret;
+                }
+            });
+
+        else if (homogenization)
+            iter.setPreProcessor(new SentencePreProcessor() {
+                @Override
+                public String preProcess(String sentence) {
+                    String ret = new InputHomogenization(sentence).transform();
+                    return ret;
+                }
+            });
+
+    }
+
+    /**
+     * Initializes this iterator with homogenization and adding labels
+     * and a batch size of 10
+     * @param vec the vector model to use
+     * @param iter the sentence iterator to use
+     * @param labels the possible labels
+     */
+    public Word2VecDataSetIterator(Word2Vec vec, LabelAwareSentenceIterator iter, List<String> labels) {
+        this(vec, iter, labels, 10);
+    }
+
+    /**
+     * Initializes this iterator with homogenization and adding labels
+     * @param vec the vector model to use
+     * @param iter the sentence iterator to use
+     * @param labels the possible labels
+     * @param batch the batch size
+     */
+    public Word2VecDataSetIterator(Word2Vec vec, LabelAwareSentenceIterator iter, List<String> labels, int batch) {
+        this(vec, iter, labels, batch, true, true);
+
+
+    }
+
+    /**
+     * Like the standard next method but allows a
+     * customizable number of examples returned
+     *
+     * @param num the number of examples
+     * @return the next data applyTransformToDestination
+     */
+    @Override
+    public DataSet next(int num) {
+        if (num <= cachedWindow.size())
+            return fromCached(num);
+        //no more sentences, return the left over
+        else if (num >= cachedWindow.size() && !iter.hasNext())
+            return fromCached(cachedWindow.size());
+
+        //need the next sentence
+        else {
+            while (cachedWindow.size() < num && iter.hasNext()) {
+                String sentence = iter.nextSentence();
+                if (sentence.isEmpty())
+                    continue;
+                List<Window> windows = Windows.windows(sentence, vec.getTokenizerFactory(), vec.getWindow(), vec);
+                if (windows.isEmpty() && !sentence.isEmpty())
+                    throw new IllegalStateException("Empty window on sentence");
+                for (Window w : windows)
+                    w.setLabel(iter.currentLabel());
+                cachedWindow.addAll(windows);
+            }
+
+            return fromCached(num);
+        }
+
+    }
+
+    private DataSet fromCached(int num) {
+        if (cachedWindow.isEmpty()) {
+            while (cachedWindow.size() < num && iter.hasNext()) {
+                String sentence = iter.nextSentence();
+                if (sentence.isEmpty())
+                    continue;
+                List<Window> windows = Windows.windows(sentence, vec.getTokenizerFactory(), vec.getWindow(), vec);
+                for (Window w : windows)
+                    w.setLabel(iter.currentLabel());
+                cachedWindow.addAll(windows);
+            }
+        }
+
+
+        List<Window> windows = new ArrayList<>(num);
+
+        for (int i = 0; i < num; i++) {
+            if (cachedWindow.isEmpty())
+                break;
+            windows.add(cachedWindow.remove(0));
+        }
+
+        if (windows.isEmpty())
+            return null;
+
+
+
+        INDArray inputs = Nd4j.create(num, inputColumns());
+        for (int i = 0; i < inputs.rows(); i++) {
+            inputs.putRow(i, WindowConverter.asExampleMatrix(windows.get(i), vec));
+        }
+
+        INDArray labelOutput = Nd4j.create(num, labels.size());
+        for (int i = 0; i < labelOutput.rows(); i++) {
+            String label = windows.get(i).getLabel();
+            labelOutput.putRow(i, FeatureUtil.toOutcomeVector(labels.indexOf(label), labels.size()));
+        }
+
+        DataSet ret = new DataSet(inputs, labelOutput);
+        if (preProcessor != null)
+            preProcessor.preProcess(ret);
+
+        return ret;
+    }
+
+    @Override
+    public int inputColumns() {
+        return vec.lookupTable().layerSize() * vec.getWindow();
+    }
+
+    @Override
+    public int totalOutcomes() {
+        return labels.size();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return false;
+    }
+
+    @Override
+    public void reset() {
+        iter.reset();
+        cachedWindow.clear();
+    }
+
+    @Override
+    public int batch() {
+        return batch;
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        this.preProcessor = preProcessor;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+
+    /**
+     * Returns {@code true} if the iteration has more elements.
+     * (In other words, returns {@code true} if {@link #next} would
+     * return an element rather than throwing an exception.)
+     *
+     * @return {@code true} if the iteration has more elements
+     */
+    @Override
+    public boolean hasNext() {
+        return iter.hasNext() || !cachedWindow.isEmpty();
+    }
+
+    /**
+     * Returns the next element in the iteration.
+     *
+     * @return the next element in the iteration
+     */
+    @Override
+    public DataSet next() {
+        return next(batch);
+    }
+
+    /**
+     * Removes from the underlying collection the last element returned
+     * by this iterator (optional operation).  This method can be called
+     * only once per call to {@link #next}.  The behavior of an iterator
+     * is unspecified if the underlying collection is modified while the
+     * iteration is in progress in any way other than by calling this
+     * method.
+     *
+     * @throws UnsupportedOperationException if the {@code remove}
+     *                                       operation is not supported by this iterator
+     * @throws IllegalStateException         if the {@code next} method has not
+     *                                       yet been called, or the {@code remove} method has already
+     *                                       been called after the last call to the {@code next}
+     *                                       method
+     */
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException();
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/HuffmanNode.java

@@ -0,0 +1,45 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.word2vec.wordstore;
+
+import lombok.Data;
+import lombok.NonNull;
+
+@Data
+public class HuffmanNode {
+    @NonNull
+    private byte[] code;
+    @NonNull
+    private int[] point;
+    private int idx;
+    private byte length;
+
+    public HuffmanNode() {
+
+    }
+
+    public HuffmanNode(byte[] code, int[] point, int index, byte length) {
+        this.code = code;
+        this.point = point;
+        this.idx = index;
+        this.length = length;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabCache.java

@@ -0,0 +1,267 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.word2vec.wordstore;
+
+
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+
+import java.io.Serializable;
+import java.util.Collection;
+
+
+public interface VocabCache<T extends SequenceElement> extends Serializable {
+
+
+    /**
+     * Load vocab
+     */
+    void loadVocab();
+
+
+    /**
+     * Vocab exists already
+     * @return
+     */
+    boolean vocabExists();
+
+    /**
+     * Saves the vocab: this allow for reuse of word frequencies	
+     */
+    void saveVocab();
+
+
+    /**
+     * Returns all of the words in the vocab
+     * @returns all the words in the vocab
+     */
+    Collection<String> words();
+
+
+    /**
+     * Increment the count for the given word
+     * @param word the word to increment the count for
+     */
+    void incrementWordCount(String word);
+
+
+    /**
+     * Increment the count for the given word by
+     * the amount increment
+     * @param word the word to increment the count for
+     * @param increment the amount to increment by
+     */
+    void incrementWordCount(String word, int increment);
+
+    /**
+     * Returns the number of times the word has occurred
+     * @param word the word to retrieve the occurrence frequency for
+     * @return 0 if hasn't occurred or the number of times
+     * the word occurs
+     */
+    int wordFrequency(String word);
+
+    /**
+     * Returns true if the cache contains the given word
+     * @param word the word to check for
+     * @return
+     */
+    boolean containsWord(String word);
+
+    /**
+     * Returns the word contained at the given index or null
+     * @param index the index of the word to get
+     * @return the word at the given index
+     */
+    String wordAtIndex(int index);
+
+    /**
+     * Returns SequenceElement at the given index or null
+     *
+     * @param index
+     * @return
+     */
+    T elementAtIndex(int index);
+
+    /**
+     * Returns the index of a given word
+     * @param word the index of a given word
+     * @return the index of a given word or -1
+     * if not found
+     */
+    int indexOf(String word);
+
+
+    /**
+     * Returns all of the vocab word nodes
+     * @return
+     */
+    Collection<T> vocabWords();
+
+
+    /**
+     * The total number of word occurrences
+     * @return the total number of word occurrences
+     */
+    long totalWordOccurrences();
+
+
+    /**
+     *
+     * @param word
+     * @return
+     */
+    T wordFor(String word);
+
+
+    T wordFor(long id);
+
+    /**
+     *
+     * @param index
+     * @param word
+     */
+    void addWordToIndex(int index, String word);
+
+
+    void addWordToIndex(int index, long elementId);
+
+    /**
+     * Inserts the word as a vocab word
+     * (it gets the vocab word from the internal token store).
+     * Note that the index must be set on the token.
+     * @param word the word to add to the vocab
+     */
+    @Deprecated
+    void putVocabWord(String word);
+
+    /**
+     * Returns the number of words in the cache
+     * @return the number of words in the cache
+     */
+    int numWords();
+
+
+    /**
+     * Count of documents a word appeared in
+     * @param word the number of documents the word appeared in
+     * @return
+     */
+    int docAppearedIn(String word);
+
+    /**
+     * Increment the document count
+     * @param word the word to increment by
+     * @param howMuch
+     */
+    void incrementDocCount(String word, long howMuch);
+
+
+    /**
+     * Set the count for the number of documents the word appears in
+     * @param word the word to set the count for
+     * @param count the count of the word
+     */
+    void setCountForDoc(String word, long count);
+
+    /**
+     * Returns the total of number of documents encountered in the corpus
+     * @return the total number of docs in the corpus
+     */
+    long totalNumberOfDocs();
+
+
+    /**
+     * Increment the doc count
+     */
+    void incrementTotalDocCount();
+
+    /**
+     * Increment the doc count
+     * @param  by the number to increment by
+     */
+    void incrementTotalDocCount(long by);
+
+    /**
+     * All of the tokens in the cache, (not necessarily apart of the vocab)
+     * @return the tokens for this cache
+     */
+    Collection<T> tokens();
+
+
+    /**
+     * Adds a token
+     * to the cache
+     * @param element the word to add
+     * @return true if token was added, false if updated
+     */
+    boolean addToken(T element);
+
+    /**
+     * Returns the token (again not necessarily in the vocab)
+     * for this word
+     * @param word the word to get the token for
+     * @return the vocab word for this token
+     */
+    T tokenFor(String word);
+
+    T tokenFor(long id);
+
+    /**
+     * Returns whether the cache
+     * contains this token or not
+     * @param token the token to tes
+     * @return whether the token exists in
+     * the cache or not
+     *
+     */
+    boolean hasToken(String token);
+
+
+    /**
+     * imports vocabulary
+     *
+     * @param vocabCache
+     */
+    void importVocabulary(VocabCache<T> vocabCache);
+
+    /**
+     * Updates counters
+     */
+    void updateWordsOccurrences();
+
+    /**
+     * Removes element with specified label from vocabulary
+     * Please note: Huffman index should be updated after element removal
+     *
+     * @param label label of the element to be removed
+     */
+    void removeElement(String label);
+
+
+    /**
+     * Removes specified element from vocabulary
+     * Please note: Huffman index should be updated after element removal
+     *
+     * @param element SequenceElement to be removed
+     */
+    void removeElement(T element);
+
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructor.java

@@ -0,0 +1,620 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.word2vec.wordstore;
+
+import lombok.Data;
+import lombok.NonNull;
+import lombok.val;
+import org.deeplearning4j.models.embeddings.WeightLookupTable;
+import org.deeplearning4j.models.embeddings.wordvectors.WordVectors;
+import org.deeplearning4j.models.sequencevectors.interfaces.SequenceIterator;
+import org.deeplearning4j.models.sequencevectors.sequence.Sequence;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.Huffman;
+import org.deeplearning4j.models.word2vec.wordstore.inmemory.AbstractCache;
+import org.deeplearning4j.text.invertedindex.InvertedIndex;
+import org.nd4j.common.util.ThreadUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.threadly.concurrent.PriorityScheduler;
+
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class VocabConstructor<T extends SequenceElement> {
+    private List<VocabSource<T>> sources = new ArrayList<>();
+    private VocabCache<T> cache;
+    private Collection<String> stopWords;
+    private boolean useAdaGrad = false;
+    private boolean fetchLabels = false;
+    private int limit;
+    private AtomicLong seqCount = new AtomicLong(0);
+    private InvertedIndex<T> index;
+    private boolean enableScavenger = false;
+    private T unk;
+    private boolean allowParallelBuilder = true;
+    private boolean lockf = false;
+
+    protected static final Logger log = LoggerFactory.getLogger(VocabConstructor.class);
+
+    private VocabConstructor() {
+
+    }
+
+    /**
+     * Placeholder for future implementation
+     * @return
+     */
+    protected WeightLookupTable<T> buildExtendedLookupTable() {
+        return null;
+    }
+
+    /**
+     * Placeholder for future implementation
+     * @return
+     */
+    protected VocabCache<T> buildExtendedVocabulary() {
+        return null;
+    }
+
+    /**
+     * This method transfers existing WordVectors model into current one
+     *
+     * @param wordVectors
+     * @return
+     */
+    @SuppressWarnings("unchecked") // method is safe, since all calls inside are using generic SequenceElement methods
+    public VocabCache<T> buildMergedVocabulary(@NonNull WordVectors wordVectors, boolean fetchLabels) {
+        return buildMergedVocabulary((VocabCache<T>) wordVectors.vocab(), fetchLabels);
+    }
+
+
+    /**
+     * This method returns total number of sequences passed through VocabConstructor
+     *
+     * @return
+     */
+    public long getNumberOfSequences() {
+        return seqCount.get();
+    }
+
+
+    /**
+     * This method transfers existing vocabulary into current one
+     *
+     * Please note: this method expects source vocabulary has Huffman tree indexes applied
+     *
+     * @param vocabCache
+     * @return
+     */
+    public VocabCache<T> buildMergedVocabulary(@NonNull VocabCache<T> vocabCache, boolean fetchLabels) {
+        if (cache == null)
+            cache = new AbstractCache.Builder<T>().build();
+        for (int t = 0; t < vocabCache.numWords(); t++) {
+            String label = vocabCache.wordAtIndex(t);
+            if (label == null)
+                continue;
+            T element = vocabCache.wordFor(label);
+
+            // skip this element if it's a label, and user don't want labels to be merged
+            if (!fetchLabels && element.isLabel())
+                continue;
+
+            cache.addToken(element);
+            cache.addWordToIndex(element.getIndex(), element.getLabel());
+
+            // backward compatibility code
+            cache.putVocabWord(element.getLabel());
+        }
+
+        if (cache.numWords() == 0)
+            throw new IllegalStateException("Source VocabCache has no indexes available, transfer is impossible");
+
+        /*
+            Now, when we have transferred vocab, we should roll over iterator, and  gather labels, if any
+         */
+        log.info("Vocab size before labels: " + cache.numWords());
+
+        if (fetchLabels) {
+            for (VocabSource<T> source : sources) {
+                SequenceIterator<T> iterator = source.getIterator();
+                iterator.reset();
+
+                while (iterator.hasMoreSequences()) {
+                    Sequence<T> sequence = iterator.nextSequence();
+                    seqCount.incrementAndGet();
+
+                    if (sequence.getSequenceLabels() != null)
+                        for (T label : sequence.getSequenceLabels()) {
+                            if (!cache.containsWord(label.getLabel())) {
+                                label.markAsLabel(true);
+                                label.setSpecial(true);
+
+                                label.setIndex(cache.numWords());
+
+                                cache.addToken(label);
+                                cache.addWordToIndex(label.getIndex(), label.getLabel());
+
+                                // backward compatibility code
+                                cache.putVocabWord(label.getLabel());
+                            }
+                        }
+                }
+            }
+        }
+
+        log.info("Vocab size after labels: " + cache.numWords());
+
+        return cache;
+    }
+
+
+    public VocabCache<T> transferVocabulary(@NonNull VocabCache<T> vocabCache, boolean buildHuffman) {
+        val result = cache != null ? cache : new AbstractCache.Builder<T>().build();
+
+        for (val v: vocabCache.tokens()) {
+                result.addToken(v);
+                // optionally transferring indices
+                if (v.getIndex() >= 0)
+                    result.addWordToIndex(v.getIndex(), v.getLabel());
+                else
+                    result.addWordToIndex(result.numWords(), v.getLabel());
+        }
+
+        if (buildHuffman) {
+            val huffman = new Huffman(result.vocabWords());
+            huffman.build();
+            huffman.applyIndexes(result);
+        }
+
+        return result;
+    }
+
+    public void processDocument(AbstractCache<T> targetVocab, Sequence<T> document,
+                                AtomicLong finalCounter, AtomicLong loopCounter) {
+        try {
+            Map<String, AtomicLong> seqMap = new HashMap<>();
+
+            if (fetchLabels && document.getSequenceLabels() != null) {
+                for (T labelWord : document.getSequenceLabels()) {
+                    if (!targetVocab.hasToken(labelWord.getLabel())) {
+                        labelWord.setSpecial(true);
+                        labelWord.markAsLabel(true);
+                        labelWord.setElementFrequency(1);
+
+                        targetVocab.addToken(labelWord);
+                    }
+                }
+            }
+
+            List<String> tokens = document.asLabels();
+            for (String token : tokens) {
+                if (stopWords != null && stopWords.contains(token))
+                    continue;
+                if (token == null || token.isEmpty())
+                    continue;
+
+                if (!targetVocab.containsWord(token)) {
+                    T element = document.getElementByLabel(token);
+                    element.setElementFrequency(1);
+                    element.setSequencesCount(1);
+                    targetVocab.addToken(element);
+                    loopCounter.incrementAndGet();
+
+                    // if there's no such element in tempHolder, it's safe to set seqCount to 1
+                    seqMap.put(token, new AtomicLong(0));
+                } else {
+                    targetVocab.incrementWordCount(token);
+
+                    // if element exists in tempHolder, we should update it seqCount, but only once per sequence
+                    if (!seqMap.containsKey(token)) {
+                        seqMap.put(token, new AtomicLong(1));
+                        T element = targetVocab.wordFor(token);
+                        element.incrementSequencesCount();
+                    }
+
+                    if (index != null) {
+                        if (document.getSequenceLabel() != null) {
+                            index.addWordsToDoc(index.numDocuments(), document.getElements(), document.getSequenceLabel());
+                        } else {
+                            index.addWordsToDoc(index.numDocuments(), document.getElements());
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        finally {
+            finalCounter.incrementAndGet();
+        }
+    }
+    /**
+     * This method scans all sources passed through builder, and returns all words as vocab.
+     * If TargetVocabCache was set during instance creation, it'll be filled too.
+     *
+     *
+     * @return
+     */
+    public VocabCache<T> buildJointVocabulary(boolean resetCounters, boolean buildHuffmanTree) {
+        long lastTime = System.currentTimeMillis();
+        long lastSequences = 0;
+        long lastElements = 0;
+        long startTime = lastTime;
+        AtomicLong parsedCount = new AtomicLong(0);
+        if (resetCounters && buildHuffmanTree)
+            throw new IllegalStateException("You can't reset counters and build Huffman tree at the same time!");
+
+        if (cache == null)
+            cache = new AbstractCache.Builder<T>().build();
+
+        log.debug("Target vocab size before building: [" + cache.numWords() + "]");
+        final AtomicLong loopCounter = new AtomicLong(0);
+
+        AbstractCache<T> topHolder = new AbstractCache.Builder<T>().minElementFrequency(0).build();
+
+        int cnt = 0;
+        int numProc = Runtime.getRuntime().availableProcessors();
+        int numThreads = Math.max(numProc / 2, 2);
+        PriorityScheduler executorService = new PriorityScheduler(numThreads);
+        final AtomicLong execCounter = new AtomicLong(0);
+        final AtomicLong finCounter = new AtomicLong(0);
+
+        for (VocabSource<T> source : sources) {
+            SequenceIterator<T> iterator = source.getIterator();
+            iterator.reset();
+
+            log.debug("Trying source iterator: [" + cnt + "]");
+            log.debug("Target vocab size before building: [" + cache.numWords() + "]");
+            cnt++;
+
+            AbstractCache<T> tempHolder = new AbstractCache.Builder<T>().build();
+
+            int sequences = 0;
+            while (iterator.hasMoreSequences()) {
+                Sequence<T> document = iterator.nextSequence();
+
+                seqCount.incrementAndGet();
+                parsedCount.addAndGet(document.size());
+                tempHolder.incrementTotalDocCount();
+                execCounter.incrementAndGet();
+
+                if (allowParallelBuilder) {
+                    executorService.execute(new VocabRunnable(tempHolder, document, finCounter, loopCounter));
+                    // as we see in profiler, this lock isn't really happen too often
+                    // we don't want too much left in tail
+
+                    while (execCounter.get() - finCounter.get() > numProc) {
+                        ThreadUtils.uncheckedSleep(1);
+                    }
+                }
+                else  {
+                    processDocument(tempHolder, document, finCounter, loopCounter);
+                }
+
+                sequences++;
+                if (seqCount.get() % 100000 == 0) {
+                    long currentTime = System.currentTimeMillis();
+                    long currentSequences = seqCount.get();
+                    long currentElements = parsedCount.get();
+
+                    double seconds = (currentTime - lastTime) / (double) 1000;
+
+
+                    double seqPerSec = (currentSequences - lastSequences) / seconds;
+                    double elPerSec = (currentElements - lastElements) / seconds;
+                    //                    log.info("Document time: {} us; hasNext time: {} us", timesNext.get(timesNext.size() / 2), timesHasNext.get(timesHasNext.size() / 2));
+                    log.info("Sequences checked: [{}]; Current vocabulary size: [{}]; Sequences/sec: {}; Words/sec: {};",
+                                    seqCount.get(), tempHolder.numWords(), String.format("%.2f", seqPerSec),
+                                    String.format("%.2f", elPerSec));
+                    lastTime = currentTime;
+                    lastElements = currentElements;
+                    lastSequences = currentSequences;
+
+
+                }
+
+                /**
+                 * Firing scavenger loop
+                 */
+                if (enableScavenger && loopCounter.get() >= 2000000 && tempHolder.numWords() > 10000000) {
+                    log.info("Starting scavenger...");
+                    while (execCounter.get() != finCounter.get()) {
+                        ThreadUtils.uncheckedSleep(1);
+                    }
+
+                    filterVocab(tempHolder, Math.max(1, source.getMinWordFrequency() / 2));
+                    loopCounter.set(0);
+                }
+
+            }
+
+            // block untill all threads are finished
+            log.debug("Waiting till all processes stop...");
+            while (execCounter.get() != finCounter.get()) {
+                ThreadUtils.uncheckedSleep(1);
+            }
+
+
+            // apply minWordFrequency set for this source
+            log.debug("Vocab size before truncation: [" + tempHolder.numWords() + "],  NumWords: ["
+                            + tempHolder.totalWordOccurrences() + "], sequences parsed: [" + seqCount.get()
+                            + "], counter: [" + parsedCount.get() + "]");
+            if (source.getMinWordFrequency() > 0) {
+                filterVocab(tempHolder, source.getMinWordFrequency());
+            }
+
+            log.debug("Vocab size after truncation: [" + tempHolder.numWords() + "],  NumWords: ["
+                            + tempHolder.totalWordOccurrences() + "], sequences parsed: [" + seqCount.get()
+                            + "], counter: [" + parsedCount.get() + "]");
+            // at this moment we're ready to transfer
+            topHolder.importVocabulary(tempHolder);
+        }
+
+        // at this moment, we have vocabulary full of words, and we have to reset counters before transfer everything back to VocabCache
+
+
+
+        System.gc();
+        cache.importVocabulary(topHolder);
+
+        // adding UNK word
+        if (unk != null) {
+            log.info("Adding UNK element to vocab...");
+            unk.setSpecial(true);
+            cache.addToken(unk);
+        }
+
+        if (resetCounters) {
+            for (T element : cache.vocabWords()) {
+                element.setElementFrequency(0);
+            }
+            cache.updateWordsOccurrences();
+        }
+
+        if (buildHuffmanTree) {
+
+            if (limit > 0) {
+                // we want to sort labels before truncating them, so we'll keep most important words
+                val words = new ArrayList<T>(cache.vocabWords());
+                Collections.sort(words);
+
+                // now rolling through them
+                for (val element : words) {
+                    if (element.getIndex() > limit && !element.isSpecial() && !element.isLabel())
+                        cache.removeElement(element.getLabel());
+                }
+            }
+            // and now we're building Huffman tree
+            val huffman = new Huffman(cache.vocabWords());
+            huffman.build();
+            huffman.applyIndexes(cache);
+        }
+
+        executorService.shutdown();
+
+        System.gc();
+
+        long endSequences = seqCount.get();
+        long endTime = System.currentTimeMillis();
+        double seconds = (endTime - startTime) / (double) 1000;
+        double seqPerSec = endSequences / seconds;
+        log.info("Sequences checked: [{}], Current vocabulary size: [{}]; Sequences/sec: [{}];", seqCount.get(),
+                        cache.numWords(), String.format("%.2f", seqPerSec));
+        return cache;
+    }
+
+    protected void filterVocab(AbstractCache<T> cache, int minWordFrequency) {
+        int numWords = cache.numWords();
+        LinkedBlockingQueue<String> labelsToRemove = new LinkedBlockingQueue<>();
+        for (T element : cache.vocabWords()) {
+            if (element.getElementFrequency() < minWordFrequency && !element.isSpecial() && !element.isLabel())
+                labelsToRemove.add(element.getLabel());
+        }
+
+        for (String label : labelsToRemove) {
+            cache.removeElement(label);
+        }
+
+        log.debug("Scavenger: Words before: {}; Words after: {};", numWords, cache.numWords());
+    }
+
+    public static class Builder<T extends SequenceElement> {
+        private List<VocabSource<T>> sources = new ArrayList<>();
+        private VocabCache<T> cache;
+        private Collection<String> stopWords = new ArrayList<>();
+        private boolean useAdaGrad = false;
+        private boolean fetchLabels = false;
+        private InvertedIndex<T> index;
+        private int limit;
+        private boolean enableScavenger = false;
+        private T unk;
+        private boolean allowParallelBuilder = true;
+        private boolean lockf = false;
+
+        public Builder() {
+
+        }
+
+        /**
+         * This method sets the limit to resulting vocabulary size.
+         *
+         * PLEASE NOTE:  This method is applicable only if huffman tree is built.
+         *
+         * @param limit
+         * @return
+         */
+        public Builder<T> setEntriesLimit(int limit) {
+            this.limit = limit;
+            return this;
+        }
+
+
+        public Builder<T> allowParallelTokenization(boolean reallyAllow) {
+            this.allowParallelBuilder = reallyAllow;
+            return this;
+        }
+
+        /**
+         * Defines, if adaptive gradients should be created during vocabulary mastering
+         *
+         * @param useAdaGrad
+         * @return
+         */
+        protected Builder<T> useAdaGrad(boolean useAdaGrad) {
+            this.useAdaGrad = useAdaGrad;
+            return this;
+        }
+
+        /**
+         * After temporary internal vocabulary is built, it will be transferred to target VocabCache you pass here
+         *
+         * @param cache target VocabCache
+         * @return
+         */
+        public Builder<T> setTargetVocabCache(@NonNull VocabCache<T> cache) {
+            this.cache = cache;
+            return this;
+        }
+
+        /**
+         * Adds SequenceIterator for vocabulary construction.
+         * Please note, you can add as many sources, as you wish.
+         *
+         * @param iterator SequenceIterator to build vocabulary from
+         * @param minElementFrequency elements with frequency below this value will be removed from vocabulary
+         * @return
+         */
+        public Builder<T> addSource(@NonNull SequenceIterator<T> iterator, int minElementFrequency) {
+            sources.add(new VocabSource<T>(iterator, minElementFrequency));
+            return this;
+        }
+
+        /*
+        public Builder<T> addSource(LabelAwareIterator iterator, int minWordFrequency) {
+            sources.add(new VocabSource(iterator, minWordFrequency));
+            return this;
+        }
+        
+        public Builder<T> addSource(SentenceIterator iterator, int minWordFrequency) {
+            sources.add(new VocabSource(new SentenceIteratorConverter(iterator), minWordFrequency));
+            return this;
+        }
+        */
+        /*
+        public Builder setTokenizerFactory(@NonNull TokenizerFactory factory) {
+            this.tokenizerFactory = factory;
+            return this;
+        }
+        */
+        public Builder<T> setStopWords(@NonNull Collection<String> stopWords) {
+            this.stopWords = stopWords;
+            return this;
+        }
+
+        /**
+         * Sets, if labels should be fetched, during vocab building
+         *
+         * @param reallyFetch
+         * @return
+         */
+        public Builder<T> fetchLabels(boolean reallyFetch) {
+            this.fetchLabels = reallyFetch;
+            return this;
+        }
+
+        public Builder<T> setIndex(InvertedIndex<T> index) {
+            this.index = index;
+            return this;
+        }
+
+        public Builder<T> enableScavenger(boolean reallyEnable) {
+            this.enableScavenger = reallyEnable;
+            return this;
+        }
+
+        public Builder<T> setUnk(T unk) {
+            this.unk = unk;
+            return this;
+        }
+
+        public VocabConstructor<T> build() {
+            VocabConstructor<T> constructor = new VocabConstructor<>();
+            constructor.sources = this.sources;
+            constructor.cache = this.cache;
+            constructor.stopWords = this.stopWords;
+            constructor.useAdaGrad = this.useAdaGrad;
+            constructor.fetchLabels = this.fetchLabels;
+            constructor.limit = this.limit;
+            constructor.index = this.index;
+            constructor.enableScavenger = this.enableScavenger;
+            constructor.unk = this.unk;
+            constructor.allowParallelBuilder = this.allowParallelBuilder;
+            constructor.lockf = this.lockf;
+
+            return constructor;
+        }
+
+        public Builder<T> setLockFactor(boolean lockf) {
+            this.lockf = lockf;
+            return this;
+        }
+    }
+
+    @Data
+    private static class VocabSource<T extends SequenceElement> {
+        @NonNull
+        private SequenceIterator<T> iterator;
+        @NonNull
+        private int minWordFrequency;
+    }
+
+
+    protected class VocabRunnable implements Runnable {
+        private final AtomicLong finalCounter;
+        private final Sequence<T> document;
+        private final AbstractCache<T> targetVocab;
+        private final AtomicLong loopCounter;
+        private AtomicBoolean done = new AtomicBoolean(false);
+
+        public VocabRunnable(@NonNull AbstractCache<T> targetVocab, @NonNull Sequence<T> sequence,
+                        @NonNull AtomicLong finalCounter, @NonNull AtomicLong loopCounter) {
+            this.finalCounter = finalCounter;
+            this.document = sequence;
+            this.targetVocab = targetVocab;
+            this.loopCounter = loopCounter;
+        }
+
+	    @Override
+        public void run() {
+             try {
+                 processDocument(targetVocab, document, finalCounter, loopCounter);
+             } catch (Exception e) {
+                 throw new RuntimeException(e);
+             }
+             finally {
+                done.set(true);
+            }
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyHolder.java

@@ -0,0 +1,613 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.word2vec.wordstore;
+
+import lombok.NonNull;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.VocabWord;
+import org.deeplearning4j.models.word2vec.wordstore.inmemory.InMemoryLookupCache;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.factory.Nd4j;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class VocabularyHolder implements Serializable {
+    private final Map<String, VocabularyWord> vocabulary = new ConcurrentHashMap<>();
+
+    // idxMap marked as transient, since there's no real reason to save this data on serialization
+    private transient Map<Integer, VocabularyWord> idxMap = new ConcurrentHashMap<>();
+    private int minWordFrequency = 0;
+    private boolean hugeModelExpected = false;
+    private int retentionDelay = 3;
+
+    private VocabCache vocabCache;
+
+    // this variable defines how often scavenger will be activated
+    private int scavengerThreshold = 2000000;
+
+    private long totalWordOccurrences = 0;
+
+    // for scavenger mechanics we need to know the actual number of words being added
+    private transient AtomicLong hiddenWordsCounter = new AtomicLong(0);
+
+    private AtomicInteger totalWordCount = new AtomicInteger(0);
+
+    private Logger logger = LoggerFactory.getLogger(VocabularyHolder.class);
+
+    private static final int MAX_CODE_LENGTH = 40;
+
+    /**
+     * Default constructor
+     */
+    protected VocabularyHolder() {
+
+    }
+
+    /**
+     * Builds VocabularyHolder from VocabCache.
+     *
+     * Basically we just ignore tokens, and transfer VocabularyWords, supposing that it's already truncated by minWordFrequency.
+     *
+     * Huffman tree data is ignored and recalculated, due to suspectable flaw in dl4j huffman impl, and it's excessive memory usage.
+     *
+     * This code is required for compatibility between dl4j w2v implementation, and standalone w2v
+     * @param cache
+     */
+    protected VocabularyHolder(@NonNull VocabCache<? extends SequenceElement> cache, boolean markAsSpecial) {
+        this.vocabCache = cache;
+        for (SequenceElement word : cache.tokens()) {
+            VocabularyWord vw = new VocabularyWord(word.getLabel());
+            vw.setCount((int) word.getElementFrequency());
+
+            // since we're importing this word from external VocabCache, we'll assume that this word is SPECIAL, and should NOT be affected by minWordFrequency
+            vw.setSpecial(markAsSpecial);
+
+            // please note: we don't transfer huffman data, since proper way is  to recalculate it after new words being added
+            if (word.getPoints() != null && !word.getPoints().isEmpty()) {
+                vw.setHuffmanNode(buildNode(word.getCodes(), word.getPoints(), word.getCodeLength(), word.getIndex()));
+            }
+
+            vocabulary.put(vw.getWord(), vw);
+        }
+
+        // there's no sense building huffman tree just for UNK word
+        if (numWords() > 1)
+            updateHuffmanCodes();
+        logger.info("Init from VocabCache is complete. " + numWords() + " word(s) were transferred.");
+    }
+
+    public static HuffmanNode buildNode(List<Byte> codes, List<Integer> points, int codeLen, int index) {
+        return new HuffmanNode(listToArray(codes), listToArray(points, MAX_CODE_LENGTH), index, (byte) codeLen);
+    }
+
+
+    public void transferBackToVocabCache() {
+        transferBackToVocabCache(this.vocabCache, true);
+    }
+
+    public void transferBackToVocabCache(VocabCache cache) {
+        transferBackToVocabCache(cache, true);
+    }
+
+    /**
+     * This method is required for compatibility purposes.
+     *  It just transfers vocabulary from VocabHolder into VocabCache
+     *
+     * @param cache
+     */
+    public void transferBackToVocabCache(VocabCache cache, boolean emptyHolder) {
+        if (!(cache instanceof InMemoryLookupCache))
+            throw new IllegalStateException("Sorry, only InMemoryLookupCache use implemented.");
+
+        // make sure that huffman codes are updated before transfer
+        List<VocabularyWord> words = words(); //updateHuffmanCodes();
+
+        for (VocabularyWord word : words) {
+            if (word.getWord().isEmpty())
+                continue;
+            VocabWord vocabWord = new VocabWord(1, word.getWord());
+
+            // if we're transferring full model, it CAN contain HistoricalGradient for AdaptiveGradient feature
+            if (word.getHistoricalGradient() != null) {
+                INDArray gradient = Nd4j.create(word.getHistoricalGradient());
+                vocabWord.setHistoricalGradient(gradient);
+            }
+
+            // put VocabWord into both Tokens and Vocabs maps
+            ((InMemoryLookupCache) cache).getVocabs().put(word.getWord(), vocabWord);
+            ((InMemoryLookupCache) cache).getTokens().put(word.getWord(), vocabWord);
+
+
+            // update Huffman tree information
+            if (word.getHuffmanNode() != null) {
+                vocabWord.setIndex(word.getHuffmanNode().getIdx());
+                vocabWord.setCodeLength(word.getHuffmanNode().getLength());
+                vocabWord.setPoints(arrayToList(word.getHuffmanNode().getPoint(), word.getHuffmanNode().getLength()));
+                vocabWord.setCodes(arrayToList(word.getHuffmanNode().getCode(), word.getHuffmanNode().getLength()));
+
+                // put word into index
+                cache.addWordToIndex(word.getHuffmanNode().getIdx(), word.getWord());
+            }
+
+            //update vocabWord counter. substract 1, since its the base value for any token
+            // >1 hack is required since VocabCache impl imples 1 as base word count, not 0
+            if (word.getCount() > 1)
+                cache.incrementWordCount(word.getWord(), word.getCount() - 1);
+        }
+
+        // at this moment its pretty safe to nullify all vocabs.
+        if (emptyHolder) {
+            idxMap.clear();
+            vocabulary.clear();
+        }
+    }
+
+    /**
+     * This method is needed ONLY for unit tests and should NOT be available in public scope.
+     *
+     * It sets the vocab size ratio, at wich dynamic scavenger will be activated
+     * @param threshold
+     */
+    protected void setScavengerActivationThreshold(int threshold) {
+        this.scavengerThreshold = threshold;
+    }
+
+
+    /**
+     *  This method is used only for VocabCache compatibility purposes
+     * @param array
+     * @param codeLen
+     * @return
+     */
+    public static List<Byte> arrayToList(byte[] array, int codeLen) {
+        List<Byte> result = new ArrayList<>();
+        for (int x = 0; x < codeLen; x++) {
+            result.add(array[x]);
+        }
+        return result;
+    }
+
+    public static byte[] listToArray(List<Byte> code) {
+        byte[] array = new byte[MAX_CODE_LENGTH];
+        for (int x = 0; x < code.size(); x++) {
+            array[x] = code.get(x);
+        }
+        return array;
+    }
+
+    public static int[] listToArray(List<Integer> points, int codeLen) {
+        int[] array = new int[points.size()];
+        for (int x = 0; x < points.size(); x++) {
+            array[x] = points.get(x).intValue();
+        }
+        return array;
+    }
+
+    /**
+     *  This method is used only for VocabCache compatibility purposes
+     * @param array
+     * @param codeLen
+     * @return
+     */
+    public static List<Integer> arrayToList(int[] array, int codeLen) {
+        List<Integer> result = new ArrayList<>();
+        for (int x = 0; x < codeLen; x++) {
+            result.add(array[x]);
+        }
+        return result;
+    }
+
+    public Collection<VocabularyWord> getVocabulary() {
+        return vocabulary.values();
+    }
+
+    public VocabularyWord getVocabularyWordByString(String word) {
+        return vocabulary.get(word);
+    }
+
+    public VocabularyWord getVocabularyWordByIdx(Integer id) {
+        return idxMap.get(id);
+    }
+
+    /**
+     * Checks vocabulary for the word existence
+     *
+     * @param word to be looked for
+     * @return TRUE of contains, FALSE otherwise
+     */
+    public boolean containsWord(String word) {
+        return vocabulary.containsKey(word);
+    }
+
+    /**
+     * Increments by one number of occurrences of the word in corpus
+     *
+     * @param word whose counter is to be incremented
+     */
+    public void incrementWordCounter(String word) {
+        if (vocabulary.containsKey(word)) {
+            vocabulary.get(word).incrementCount();
+        }
+        // there's no need to throw such exception here. just do nothing if word is not found
+        //else throw new IllegalStateException("No such word found");
+    }
+
+    /**
+     * Adds new word to vocabulary
+     *
+     * @param word to be added
+     */
+    // TODO: investigate, if it's worth to make this internally synchronized and virtually thread-safe
+    public void addWord(String word) {
+        if (!vocabulary.containsKey(word)) {
+            VocabularyWord vw = new VocabularyWord(word);
+
+            /*
+                TODO: this should be done in different way, since this implementation causes minWordFrequency ultimate ignoral if markAsSpecial set to TRUE
+            
+                Probably the best way to solve it, is remove markAsSpecial option here, and let this issue be regulated with minWordFrequency
+              */
+            // vw.setSpecial(markAsSpecial);
+
+            // initialize frequencyShift only if hugeModelExpected. It's useless otherwise :)
+            if (hugeModelExpected)
+                vw.setFrequencyShift(new byte[retentionDelay]);
+
+            vocabulary.put(word, vw);
+
+
+
+            if (hugeModelExpected && minWordFrequency > 1
+                            && hiddenWordsCounter.incrementAndGet() % scavengerThreshold == 0)
+                activateScavenger();
+
+            return;
+        }
+    }
+
+    public void addWord(VocabularyWord word) {
+        vocabulary.put(word.getWord(), word);
+    }
+
+    public void consumeVocabulary(VocabularyHolder holder) {
+        for (VocabularyWord word : holder.getVocabulary()) {
+            if (!this.containsWord(word.getWord())) {
+                this.addWord(word);
+            } else {
+                holder.incrementWordCounter(word.getWord());
+            }
+        }
+    }
+
+    /**
+     * This method removes low-frequency words based on their frequency change between activations.
+     * I.e. if word has appeared only once, and it's retained the same frequency over consequence activations, we can assume it can be removed freely
+     */
+    protected synchronized void activateScavenger() {
+        int initialSize = vocabulary.size();
+        List<VocabularyWord> words = new ArrayList<>(vocabulary.values());
+        for (VocabularyWord word : words) {
+            // scavenging could be applied only to non-special tokens that are below minWordFrequency
+            if (word.isSpecial() || word.getCount() >= minWordFrequency || word.getFrequencyShift() == null) {
+                word.setFrequencyShift(null);
+                continue;
+            }
+
+            // save current word counter to byte array at specified position
+            word.getFrequencyShift()[word.getRetentionStep()] = (byte) word.getCount();
+
+            /*
+                    we suppose that we're hunting only low-freq words that already passed few activations
+                    so, we assume word personal threshold as 20% of minWordFrequency, but not less then 1.
+            
+                    so, if after few scavenging cycles wordCount is still <= activation - just remove word.
+                    otherwise nullify word.frequencyShift to avoid further checks
+              */
+            int activation = Math.max(minWordFrequency / 5, 2);
+            logger.debug("Current state> Activation: [" + activation + "], retention info: "
+                            + Arrays.toString(word.getFrequencyShift()));
+            if (word.getCount() <= activation && word.getFrequencyShift()[this.retentionDelay - 1] > 0) {
+
+                // if final word count at latest retention point is the same as at the beginning - just remove word
+                if (word.getFrequencyShift()[this.retentionDelay - 1] <= activation
+                                && word.getFrequencyShift()[this.retentionDelay - 1] == word.getFrequencyShift()[0]) {
+                    vocabulary.remove(word.getWord());
+                }
+            }
+
+            // shift retention history to the left
+            if (word.getRetentionStep() < retentionDelay - 1) {
+                word.incrementRetentionStep();
+            } else {
+                for (int x = 1; x < retentionDelay; x++) {
+                    word.getFrequencyShift()[x - 1] = word.getFrequencyShift()[x];
+                }
+            }
+        }
+        logger.info("Scavenger was activated. Vocab size before: [" + initialSize + "],  after: [" + vocabulary.size()
+                        + "]");
+    }
+
+    /**
+     * This methods reset counters for all words in vocabulary
+     */
+    public void resetWordCounters() {
+        for (VocabularyWord word : getVocabulary()) {
+            word.setHuffmanNode(null);
+            word.setFrequencyShift(null);
+            word.setCount(0);
+        }
+    }
+
+    /**
+     *
+     * @return number of words in vocabulary
+     */
+    public int numWords() {
+        return vocabulary.size();
+    }
+
+    /**
+     * The same as truncateVocabulary(this.minWordFrequency)
+     */
+    public void truncateVocabulary() {
+        truncateVocabulary(minWordFrequency);
+    }
+
+    /**
+     * All words with frequency below threshold wii be removed
+     *
+     * @param threshold exclusive threshold for removal
+     */
+    public void truncateVocabulary(int threshold) {
+        logger.debug("Truncating vocabulary to minWordFrequency: [" + threshold + "]");
+        Set<String> keyset = vocabulary.keySet();
+        for (String word : keyset) {
+            VocabularyWord vw = vocabulary.get(word);
+
+            // please note: we're not applying threshold to SPECIAL words
+            if (!vw.isSpecial() && vw.getCount() < threshold) {
+                vocabulary.remove(word);
+                if (vw.getHuffmanNode() != null)
+                    idxMap.remove(vw.getHuffmanNode().getIdx());
+            }
+        }
+    }
+
+    /**
+     * build binary tree ordered by counter.
+     *
+     * Based on original w2v by google
+     */
+    public List<VocabularyWord> updateHuffmanCodes() {
+        int min1i;
+        int min2i;
+        int b;
+        int i;
+        // get vocabulary as sorted list
+        List<VocabularyWord> vocab = this.words();
+        int count[] = new int[vocab.size() * 2 + 1];
+        int parent_node[] = new int[vocab.size() * 2 + 1];
+        byte binary[] = new byte[vocab.size() * 2 + 1];
+
+        // at this point vocab is sorted, with descending order
+        for (int a = 0; a < vocab.size(); a++)
+            count[a] = vocab.get(a).getCount();
+        for (int a = vocab.size(); a < vocab.size() * 2; a++)
+            count[a] = Integer.MAX_VALUE;
+        int pos1 = vocab.size() - 1;
+        int pos2 = vocab.size();
+        for (int a = 0; a < vocab.size(); a++) {
+            // First, find two smallest nodes 'min1, min2'
+            if (pos1 >= 0) {
+                if (count[pos1] < count[pos2]) {
+                    min1i = pos1;
+                    pos1--;
+                } else {
+                    min1i = pos2;
+                    pos2++;
+                }
+            } else {
+                min1i = pos2;
+                pos2++;
+            }
+            if (pos1 >= 0) {
+                if (count[pos1] < count[pos2]) {
+                    min2i = pos1;
+                    pos1--;
+                } else {
+                    min2i = pos2;
+                    pos2++;
+                }
+            } else {
+                min2i = pos2;
+                pos2++;
+            }
+            count[vocab.size() + a] = count[min1i] + count[min2i];
+            parent_node[min1i] = vocab.size() + a;
+            parent_node[min2i] = vocab.size() + a;
+            binary[min2i] = 1;
+        }
+
+        // Now assign binary code to each vocabulary word
+        byte[] code = new byte[MAX_CODE_LENGTH];
+        int[] point = new int[MAX_CODE_LENGTH];
+
+        for (int a = 0; a < vocab.size(); a++) {
+            b = a;
+            i = 0;
+            byte[] lcode = new byte[MAX_CODE_LENGTH];
+            int[] lpoint = new int[MAX_CODE_LENGTH];
+            while (true) {
+                code[i] = binary[b];
+                point[i] = b;
+                i++;
+                b = parent_node[b];
+                if (b == vocab.size() * 2 - 2)
+                    break;
+            }
+
+            lpoint[0] = vocab.size() - 2;
+            for (b = 0; b < i; b++) {
+                lcode[i - b - 1] = code[b];
+                lpoint[i - b] = point[b] - vocab.size();
+            }
+
+            vocab.get(a).setHuffmanNode(new HuffmanNode(lcode, lpoint, a, (byte) i));
+        }
+
+        idxMap.clear();
+        for (VocabularyWord word : vocab) {
+            idxMap.put(word.getHuffmanNode().getIdx(), word);
+        }
+
+        return vocab;
+    }
+
+    /**
+     * This method returns index of word in sorted list.
+     *
+     * @param word
+     * @return
+     */
+    public int indexOf(String word) {
+        if (vocabulary.containsKey(word)) {
+            return vocabulary.get(word).getHuffmanNode().getIdx();
+        } else
+            return -1;
+    }
+
+
+    /**
+     * Returns sorted list of words in vocabulary.
+     * Sort is DESCENDING.
+     *
+     * @return list of VocabularyWord
+     */
+    public List<VocabularyWord> words() {
+        List<VocabularyWord> vocab = new ArrayList<>(vocabulary.values());
+        Collections.sort(vocab, new Comparator<VocabularyWord>() {
+            @Override
+            public int compare(VocabularyWord o1, VocabularyWord o2) {
+                return Integer.compare(o2.getCount(), o1.getCount());
+            }
+        });
+
+        return vocab;
+    }
+
+    public long totalWordsBeyondLimit() {
+        if (totalWordOccurrences == 0) {
+            for (VocabularyWord word : vocabulary.values()) {
+                totalWordOccurrences += word.getCount();
+            }
+            return totalWordOccurrences;
+        } else
+            return totalWordOccurrences;
+    }
+
+    public static class Builder {
+        private VocabCache cache = null;
+        private int minWordFrequency = 0;
+        private boolean hugeModelExpected = false;
+        private int scavengerThreshold = 2000000;
+        private int retentionDelay = 3;
+
+        public Builder() {
+
+        }
+
+        public Builder externalCache(@NonNull VocabCache cache) {
+            this.cache = cache;
+            return this;
+        }
+
+        public Builder minWordFrequency(int threshold) {
+            this.minWordFrequency = threshold;
+            return this;
+        }
+
+        /**
+         * With this argument set to true, you'll have your vocab scanned for low-freq words periodically.
+         *
+         * Please note: this is incompatible with SPECIAL mechanics.
+         *
+         * @param reallyExpected
+         * @return
+         */
+        public Builder hugeModelExpected(boolean reallyExpected) {
+            this.hugeModelExpected = reallyExpected;
+            return this;
+        }
+
+        /**
+         *  Activation threshold defines, how ofter scavenger will be executed, to throw away low-frequency keywords.
+         *  Good values to start mostly depends on your workstation. Something like 1000000 looks pretty nice to start with.
+         *  Too low values can lead to undesired removal of words from vocab.
+         *
+         *  Please note: this is incompatible with SPECIAL mechanics.
+         *
+         * @param threshold
+         * @return
+         */
+        public Builder scavengerActivationThreshold(int threshold) {
+            this.scavengerThreshold = threshold;
+            return this;
+        }
+
+        /**
+         * Retention delay defines, how long low-freq word will be kept in vocab, during building.
+         * Good values to start with: 3,4,5. Not too high, and not too low.
+         *
+         * Please note: this is incompatible with SPECIAL mechanics.
+         *
+         * @param delay
+         * @return
+         */
+        public Builder scavengerRetentionDelay(int delay) {
+            if (delay < 2)
+                throw new IllegalStateException("Delay < 2 doesn't really makes sense");
+            this.retentionDelay = delay;
+            return this;
+        }
+
+        public VocabularyHolder build() {
+            VocabularyHolder holder = null;
+            if (cache != null) {
+                holder = new VocabularyHolder(cache, true);
+            } else {
+                holder = new VocabularyHolder();
+            }
+            holder.minWordFrequency = this.minWordFrequency;
+            holder.hugeModelExpected = this.hugeModelExpected;
+            holder.scavengerThreshold = this.scavengerThreshold;
+            holder.retentionDelay = this.retentionDelay;
+
+            return holder;
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyWord.java

@@ -0,0 +1,132 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.word2vec.wordstore;
+
+import lombok.Data;
+import lombok.NonNull;
+import org.nd4j.shade.jackson.databind.DeserializationFeature;
+import org.nd4j.shade.jackson.databind.MapperFeature;
+import org.nd4j.shade.jackson.databind.ObjectMapper;
+import org.nd4j.shade.jackson.databind.SerializationFeature;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+@Data
+public class VocabularyWord implements Serializable {
+    @NonNull
+    private String word;
+    private int count = 1;
+
+    // these fields are used for vocab serialization/deserialization only. Usual runtime value is null.
+    private double[] syn0;
+    private double[] syn1;
+    private double[] syn1Neg;
+    private double[] historicalGradient;
+
+    private static ObjectMapper mapper;
+    private static final Object lock = new Object();
+
+    // There's no reasons to save HuffmanNode data, it will be recalculated after deserialization
+    private transient HuffmanNode huffmanNode;
+
+    // empty constructor is required for proper deserialization
+    public VocabularyWord() {
+
+    }
+
+    public VocabularyWord(@NonNull String word) {
+        this.word = word;
+    }
+
+    /*
+        since scavenging mechanics are targeting low-freq words, byte values is definitely enough.
+        please note, array is initialized outside of this scope, since it's only holder, no logic involved inside this class
+      */
+    private transient byte[] frequencyShift;
+    private transient byte retentionStep;
+
+    // special mark means that this word should NOT be affected by minWordFrequency setting
+    private boolean special = false;
+
+    public void incrementCount() {
+        this.count++;
+    }
+
+    public void incrementRetentionStep() {
+        this.retentionStep += 1;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        VocabularyWord word1 = (VocabularyWord) o;
+
+        return word != null ? word.equals(word1.word) : word1.word == null;
+
+    }
+
+    @Override
+    public int hashCode() {
+        return word != null ? word.hashCode() : 0;
+    }
+
+    private static ObjectMapper mapper() {
+        if (mapper == null) {
+            synchronized (lock) {
+                if (mapper == null) {
+                    mapper = new ObjectMapper();
+                    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+                    mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
+                    mapper.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true);
+                    return mapper;
+                }
+            }
+        }
+        return mapper;
+    }
+
+    public String toJson() {
+        ObjectMapper mapper = mapper();
+        try {
+            /*
+                we need JSON as single line to save it at first line of the CSV model file
+            */
+            return mapper.writeValueAsString(this);
+        } catch (org.nd4j.shade.jackson.core.JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static VocabularyWord fromJson(String json) {
+        ObjectMapper mapper = mapper();
+        try {
+            VocabularyWord ret = mapper.readValue(json, VocabularyWord.class);
+            return ret;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/AbstractCache.java

@@ -0,0 +1,760 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.word2vec.wordstore.inmemory;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement;
+import org.deeplearning4j.models.word2vec.VocabWord;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.nd4j.shade.jackson.annotation.JsonAutoDetect;
+import org.nd4j.shade.jackson.annotation.JsonTypeInfo;
+import org.nd4j.shade.jackson.core.JsonProcessingException;
+import org.nd4j.shade.jackson.databind.*;
+import org.nd4j.shade.jackson.databind.type.CollectionType;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Slf4j
+@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class")
+@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE,
+        setterVisibility = JsonAutoDetect.Visibility.NONE)
+public class AbstractCache<T extends SequenceElement> implements VocabCache<T> {
+    private static final String CLASS_FIELD = "@class";
+    private static final String VOCAB_LIST_FIELD = "VocabList";
+    private static final String VOCAB_ITEM_FIELD = "VocabItem";
+    private static final String DOC_CNT_FIELD = "DocumentsCounter";
+    private static final String MINW_FREQ_FIELD = "MinWordsFrequency";
+    private static final String HUGE_MODEL_FIELD = "HugeModelExpected";
+    private static final String STOP_WORDS_FIELD = "StopWords";
+    private static final String SCAVENGER_FIELD = "ScavengerThreshold";
+    private static final String RETENTION_FIELD = "RetentionDelay";
+    private static final String TOTAL_WORD_FIELD = "TotalWordCount";
+
+    private final ConcurrentMap<Long, T> vocabulary = new ConcurrentHashMap<>();
+
+    private final Map<String, T> extendedVocabulary = new ConcurrentHashMap<>();
+
+    private final Map<Integer, T> idxMap = new ConcurrentHashMap<>();
+
+    private final AtomicLong documentsCounter = new AtomicLong(0);
+
+
+    private int minWordFrequency = 0;
+    private boolean hugeModelExpected = false;
+
+    // we're using <String>for compatibility & failproof reasons: it's easier to store unique labels then abstract objects of unknown size
+    // TODO: wtf this one is doing here?
+    private List<String> stopWords = new ArrayList<>(); // stop words
+
+    // this variable defines how often scavenger will be activated
+    private int scavengerThreshold = 3000000; // ser
+    private int retentionDelay = 3; // ser
+
+    // for scavenger mechanics we need to know the actual number of words being added
+    private transient AtomicLong hiddenWordsCounter = new AtomicLong(0);
+
+    private final AtomicLong totalWordCount = new AtomicLong(0); // ser
+
+    private static final int MAX_CODE_LENGTH = 40;
+
+    /**
+     * Deserialize vocabulary from specified path
+     */
+    @Override
+    public void loadVocab() {
+        // TODO: this method should be static and accept path
+    }
+
+    /**
+     * Returns true, if number of elements in vocabulary > 0, false otherwise
+     *
+     * @return
+     */
+    @Override
+    public boolean vocabExists() {
+        return !vocabulary.isEmpty();
+    }
+
+    /**
+     * Serialize vocabulary to specified path
+     *
+     */
+    @Override
+    public void saveVocab() {
+        // TODO: this method should be static and accept path
+    }
+
+    /**
+     * Returns collection of labels available in this vocabulary
+     *
+     * @return
+     */
+    @Override
+    public Collection<String> words() {
+        return Collections.unmodifiableCollection(extendedVocabulary.keySet());
+    }
+
+    /**
+     * Increment frequency for specified label by 1
+     *
+     * @param word the word to increment the count for
+     */
+    @Override
+    public void incrementWordCount(String word) {
+        incrementWordCount(word, 1);
+    }
+
+
+    /**
+     * Increment frequency for specified label by specified value
+     *
+     * @param word the word to increment the count for
+     * @param increment the amount to increment by
+     */
+    @Override
+    public void incrementWordCount(String word, int increment) {
+        T element = extendedVocabulary.get(word);
+        if (element != null) {
+            element.increaseElementFrequency(increment);
+            totalWordCount.addAndGet(increment);
+        }
+    }
+
+    /**
+     * Returns the SequenceElement's frequency over training corpus
+     *
+     * @param word the word to retrieve the occurrence frequency for
+     * @return
+     */
+    @Override
+    public int wordFrequency(@NonNull String word) {
+        // TODO: proper wordFrequency impl should return long, instead of int
+        T element = extendedVocabulary.get(word);
+        if (element != null)
+            return (int) element.getElementFrequency();
+        return 0;
+    }
+
+    /**
+     * Checks, if specified label exists in vocabulary
+     *
+     * @param word the word to check for
+     * @return
+     */
+    @Override
+    public boolean containsWord(String word) {
+        return extendedVocabulary.containsKey(word);
+    }
+
+    /**
+     * Checks, if specified element exists in vocabulary
+     *
+     * @param element
+     * @return
+     */
+    public boolean containsElement(T element) {
+        // FIXME: lolwtf
+        return vocabulary.values().contains(element);
+    }
+
+    /**
+     * Returns the label of the element at specified Huffman index
+     *
+     * @param index the index of the word to get
+     * @return
+     */
+    @Override
+    public String wordAtIndex(int index) {
+        T element = idxMap.get(index);
+        if (element != null) {
+            return element.getLabel();
+        }
+        return null;
+    }
+
+    /**
+     * Returns SequenceElement at specified index
+     *
+     * @param index
+     * @return
+     */
+    @Override
+    public T elementAtIndex(int index) {
+        return idxMap.get(index);
+    }
+
+    /**
+     * Returns Huffman index for specified label
+     *
+     * @param label the label to get index for
+     * @return >=0 if label exists, -1 if Huffman tree wasn't built yet, -2 if specified label wasn't found
+     */
+    @Override
+    public int indexOf(String label) {
+        T token = tokenFor(label);
+        if (token != null) {
+            return token.getIndex();
+        } else
+            return -2;
+    }
+
+    /**
+     * Returns collection of SequenceElements stored in this vocabulary
+     *
+     * @return
+     */
+    @Override
+    public Collection<T> vocabWords() {
+        return vocabulary.values();
+    }
+
+    /**
+     * Returns total number of elements observed
+     *
+     * @return
+     */
+    @Override
+    public long totalWordOccurrences() {
+        return totalWordCount.get();
+    }
+
+    public void setTotalWordOccurences(long value) {
+        totalWordCount.set(value);
+    }
+
+    /**
+     * Returns SequenceElement for specified label
+     *
+     * @param label to fetch element for
+     * @return
+     */
+    @Override
+    public T wordFor(@NonNull String label) {
+        return extendedVocabulary.get(label);
+    }
+
+    @Override
+    public T wordFor(long id) {
+        return vocabulary.get(id);
+    }
+
+    /**
+     * This method allows to insert specified label to specified Huffman tree position.
+     * CAUTION: Never use this, unless you 100% sure what are you doing.
+     *
+     * @param index
+     * @param label
+     */
+    @Override
+    public void addWordToIndex(int index, String label) {
+        if (index >= 0) {
+            T token = tokenFor(label);
+            if (token != null) {
+                idxMap.put(index, token);
+                token.setIndex(index);
+            }
+        }
+    }
+
+    @Override
+    public void addWordToIndex(int index, long elementId) {
+        if (index >= 0)
+            idxMap.put(index, tokenFor(elementId));
+    }
+
+    @Override
+    @Deprecated
+    public void putVocabWord(String word) {
+        if (!containsWord(word))
+            throw new IllegalStateException("Specified label is not present in vocabulary");
+    }
+
+    /**
+     * Returns number of elements in this vocabulary
+     *
+     * @return
+     */
+    @Override
+    public int numWords() {
+        return vocabulary.size();
+    }
+
+    /**
+     * Returns number of documents (if applicable) the label was observed in.
+     *
+     * @param word the number of documents the word appeared in
+     * @return
+     */
+    @Override
+    public int docAppearedIn(String word) {
+        T element = extendedVocabulary.get(word);
+        if (element != null) {
+            return (int) element.getSequencesCount();
+        } else
+            return -1;
+    }
+
+    /**
+     * Increment number of documents the label was observed in
+     *
+     * Please note: this method is NOT thread-safe
+     *
+     * @param word the word to increment by
+     * @param howMuch
+     */
+    @Override
+    public void incrementDocCount(String word, long howMuch) {
+        T element = extendedVocabulary.get(word);
+        if (element != null) {
+            element.incrementSequencesCount();
+        }
+    }
+
+    /**
+     * Set exact number of observed documents that contain specified word
+     *
+     * Please note: this method is NOT thread-safe
+     *
+     * @param word the word to set the count for
+     * @param count the count of the word
+     */
+    @Override
+    public void setCountForDoc(String word, long count) {
+        T element = extendedVocabulary.get(word);
+        if (element != null) {
+            element.setSequencesCount(count);
+        }
+    }
+
+    /**
+     * Returns total number of documents observed (if applicable)
+     *
+     * @return
+     */
+    @Override
+    public long totalNumberOfDocs() {
+        return documentsCounter.intValue();
+    }
+
+    /**
+     * Increment total number of documents observed by 1
+     */
+    @Override
+    public void incrementTotalDocCount() {
+        documentsCounter.incrementAndGet();
+    }
+
+    /**
+     * Increment total number of documents observed by specified value
+     */
+    @Override
+    public void incrementTotalDocCount(long by) {
+        documentsCounter.addAndGet(by);
+    }
+
+    /**
+     * This method allows to set total number of documents
+     * @param by
+     */
+    public void setTotalDocCount(long by) {
+
+        documentsCounter.set(by);
+    }
+
+
+    /**
+     * Returns collection of SequenceElements from this vocabulary. The same as vocabWords() method
+     *
+     * @return collection of SequenceElements
+     */
+    @Override
+    public Collection<T> tokens() {
+        return vocabWords();
+    }
+
+    /**
+     * This method adds specified SequenceElement to vocabulary
+     *
+     * @param element the word to add
+     */
+    @Override
+    public boolean addToken(T element) {
+        boolean ret = false;
+        T oldElement = vocabulary.putIfAbsent(element.getStorageId(), element);
+        if (oldElement == null) {
+            //putIfAbsent added our element
+            if (element.getLabel() != null) {
+                extendedVocabulary.put(element.getLabel(), element);
+            }
+            oldElement = element;
+            ret = true;
+        } else {
+            oldElement.incrementSequencesCount(element.getSequencesCount());
+            oldElement.increaseElementFrequency((int) element.getElementFrequency());
+        }
+        totalWordCount.addAndGet((long) oldElement.getElementFrequency());
+        return ret;
+    }
+
+    public void addToken(T element, boolean lockf) {
+        T oldElement = vocabulary.putIfAbsent(element.getStorageId(), element);
+        if (oldElement == null) {
+            //putIfAbsent added our element
+            if (element.getLabel() != null) {
+                extendedVocabulary.put(element.getLabel(), element);
+            }
+            oldElement = element;
+        } else {
+            oldElement.incrementSequencesCount(element.getSequencesCount());
+            oldElement.increaseElementFrequency((int) element.getElementFrequency());
+        }
+        totalWordCount.addAndGet((long) oldElement.getElementFrequency());
+    }
+
+    /**
+     * Returns SequenceElement for specified label. The same as wordFor() method.
+     *
+     * @param label the label to get the token for
+     * @return
+     */
+    @Override
+    public T tokenFor(String label) {
+        return wordFor(label);
+    }
+
+    @Override
+    public T tokenFor(long id) {
+        return vocabulary.get(id);
+    }
+
+    /**
+     * Checks, if specified label already exists in vocabulary. The same as containsWord() method.
+     *
+     * @param label the token to test
+     * @return
+     */
+    @Override
+    public boolean hasToken(String label) {
+        return containsWord(label);
+    }
+
+
+    /**
+     * This method imports all elements from VocabCache passed as argument
+     * If element already exists,
+     *
+     * @param vocabCache
+     */
+    public void importVocabulary(@NonNull VocabCache<T> vocabCache) {
+        AtomicBoolean added = new AtomicBoolean(false);
+        for (T element : vocabCache.vocabWords()) {
+            if (this.addToken(element))
+                added.set(true);
+        }
+        //logger.info("Current state: {}; Adding value: {}", this.documentsCounter.get(), vocabCache.totalNumberOfDocs());
+        if (added.get())
+            this.documentsCounter.addAndGet(vocabCache.totalNumberOfDocs());
+    }
+
+    @Override
+    public void updateWordsOccurrences() {
+        totalWordCount.set(0);
+        for (T element : vocabulary.values()) {
+            long value = (long) element.getElementFrequency();
+
+            if (value > 0) {
+                totalWordCount.addAndGet(value);
+            }
+        }
+        log.info("Updated counter: [" + totalWordCount.get() + "]");
+    }
+
+    @Override
+    public void removeElement(String label) {
+        SequenceElement element = extendedVocabulary.get(label);
+        if (element != null) {
+            totalWordCount.getAndAdd((long) element.getElementFrequency() * -1);
+            idxMap.remove(element.getIndex());
+            extendedVocabulary.remove(label);
+            vocabulary.remove(element.getStorageId());
+        } else
+            throw new IllegalStateException("Can't get label: '" + label + "'");
+    }
+
+    @Override
+    public void removeElement(T element) {
+        removeElement(element.getLabel());
+    }
+
+    private static ObjectMapper mapper = null;
+    private static final Object lock = new Object();
+
+    private static ObjectMapper mapper() {
+        if (mapper == null) {
+            synchronized (lock) {
+                if (mapper == null) {
+                    mapper = new ObjectMapper();
+                    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+                    mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
+                    return mapper;
+                }
+            }
+        }
+        return mapper;
+    }
+
+    public String toJson() throws JsonProcessingException {
+
+        JsonObject retVal = new JsonObject();
+        ObjectMapper mapper = mapper();
+        Iterator<T> iter = vocabulary.values().iterator();
+        Class clazz = null;
+        if (iter.hasNext())
+            clazz = iter.next().getClass();
+        else
+            return retVal.getAsString();
+
+        retVal.addProperty(CLASS_FIELD, mapper.writeValueAsString(this.getClass().getName()));
+
+        JsonArray jsonValues = new JsonArray();
+        for (T value : vocabulary.values()) {
+            JsonObject item = new JsonObject();
+            item.addProperty(CLASS_FIELD, mapper.writeValueAsString(clazz));
+            item.addProperty(VOCAB_ITEM_FIELD, mapper.writeValueAsString(value));
+            jsonValues.add(item);
+        }
+        retVal.add(VOCAB_LIST_FIELD, jsonValues);
+
+        retVal.addProperty(DOC_CNT_FIELD, mapper.writeValueAsString(documentsCounter.longValue()));
+        retVal.addProperty(MINW_FREQ_FIELD, mapper.writeValueAsString(minWordFrequency));
+        retVal.addProperty(HUGE_MODEL_FIELD, mapper.writeValueAsString(hugeModelExpected));
+
+        retVal.addProperty(STOP_WORDS_FIELD, mapper.writeValueAsString(stopWords));
+
+        retVal.addProperty(SCAVENGER_FIELD, mapper.writeValueAsString(scavengerThreshold));
+        retVal.addProperty(RETENTION_FIELD, mapper.writeValueAsString(retentionDelay));
+        retVal.addProperty(TOTAL_WORD_FIELD, mapper.writeValueAsString(totalWordCount.longValue()));
+
+        return retVal.toString();
+    }
+
+    public static <T extends SequenceElement> AbstractCache<T> fromJson(String jsonString)  throws IOException {
+        AbstractCache<T> retVal = new AbstractCache.Builder<T>().build();
+
+        JsonParser parser = new JsonParser();
+        JsonObject json = parser.parse(jsonString).getAsJsonObject();
+
+        ObjectMapper mapper = mapper();
+
+        CollectionType wordsCollectionType = mapper.getTypeFactory()
+                .constructCollectionType(List.class, VocabWord.class);
+
+        List<T> items = new ArrayList<>();
+        JsonArray jsonArray = json.get(VOCAB_LIST_FIELD).getAsJsonArray();
+        for (int i = 0; i < jsonArray.size(); ++i) {
+            VocabWord item = mapper.readValue(jsonArray.get(i).getAsJsonObject().get(VOCAB_ITEM_FIELD).getAsString(), VocabWord.class);
+            items.add((T)item);
+        }
+
+        ConcurrentMap<Long, T> vocabulary = new ConcurrentHashMap<>();
+        Map<String, T> extendedVocabulary = new ConcurrentHashMap<>();
+        Map<Integer, T> idxMap = new ConcurrentHashMap<>();
+
+        for (T item : items) {
+            vocabulary.put(item.getStorageId(), item);
+            extendedVocabulary.put(item.getLabel(), item);
+            idxMap.put(item.getIndex(), item);
+        }
+        List<String> stopWords = mapper.readValue(json.get(STOP_WORDS_FIELD).getAsString(), List.class);
+
+        Long documentsCounter = json.get(DOC_CNT_FIELD).getAsLong();
+        Integer minWordsFrequency = json.get(MINW_FREQ_FIELD).getAsInt();
+        Boolean hugeModelExpected = json.get(HUGE_MODEL_FIELD).getAsBoolean();
+        Integer scavengerThreshold = json.get(SCAVENGER_FIELD).getAsInt();
+        Integer retentionDelay = json.get(RETENTION_FIELD).getAsInt();
+        Long totalWordCount = json.get(TOTAL_WORD_FIELD).getAsLong();
+
+        retVal.vocabulary.putAll(vocabulary);
+        retVal.extendedVocabulary.putAll(extendedVocabulary);
+        retVal.idxMap.putAll(idxMap);
+        retVal.stopWords.addAll(stopWords);
+        retVal.documentsCounter.set(documentsCounter);
+        retVal.minWordFrequency = minWordsFrequency;
+        retVal.hugeModelExpected = hugeModelExpected;
+        retVal.scavengerThreshold = scavengerThreshold;
+        retVal.retentionDelay = retentionDelay;
+        retVal.totalWordCount.set(totalWordCount);
+        return retVal;
+    }
+
+    public static class Builder<T extends SequenceElement> {
+        protected int scavengerThreshold = 3000000;
+        protected int retentionDelay = 3;
+        protected int minElementFrequency;
+        protected boolean hugeModelExpected = false;
+
+
+        public Builder<T> hugeModelExpected(boolean reallyExpected) {
+            this.hugeModelExpected = reallyExpected;
+            return this;
+        }
+
+        public Builder<T> scavengerThreshold(int threshold) {
+            this.scavengerThreshold = threshold;
+            return this;
+        }
+
+        public Builder<T> scavengerRetentionDelay(int delay) {
+            this.retentionDelay = delay;
+            return this;
+        }
+
+        public Builder<T> minElementFrequency(int minFrequency) {
+            this.minElementFrequency = minFrequency;
+            return this;
+        }
+
+        public AbstractCache<T> build() {
+            AbstractCache<T> cache = new AbstractCache<>();
+            cache.minWordFrequency = this.minElementFrequency;
+            cache.scavengerThreshold = this.scavengerThreshold;
+            cache.retentionDelay = this.retentionDelay;
+
+            return cache;
+        }
+
+    }
+
+    @Override
+    public String toString() {
+        return "AbstractCache{" +
+                "vocabulary=" + vocabulary +
+                ", extendedVocabulary=" + extendedVocabulary +
+                ", idxMap=" + idxMap +
+                ", documentsCounter=" + documentsCounter +
+                ", minWordFrequency=" + minWordFrequency +
+                ", hugeModelExpected=" + hugeModelExpected +
+                ", stopWords=" + stopWords +
+                ", scavengerThreshold=" + scavengerThreshold +
+                ", retentionDelay=" + retentionDelay +
+                ", hiddenWordsCounter=" + hiddenWordsCounter +
+                ", totalWordCount=" + totalWordCount +
+                '}';
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof AbstractCache)) return false;
+        AbstractCache<?> that = (AbstractCache<?>) o;
+        return getMinWordFrequency() == that.getMinWordFrequency() && isHugeModelExpected() == that.isHugeModelExpected() && getScavengerThreshold() == that.getScavengerThreshold() && getRetentionDelay() == that.getRetentionDelay() && Objects.equals(getVocabulary(), that.getVocabulary()) && Objects.equals(getExtendedVocabulary(), that.getExtendedVocabulary()) && Objects.equals(getIdxMap(), that.getIdxMap()) && Objects.equals(getStopWords(), that.getStopWords());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getVocabulary(), getExtendedVocabulary(), getIdxMap(), getMinWordFrequency(), isHugeModelExpected(), getStopWords(), getScavengerThreshold(), getRetentionDelay());
+    }
+
+    public ConcurrentMap<Long, T> getVocabulary() {
+        return vocabulary;
+    }
+
+    public Map<String, T> getExtendedVocabulary() {
+        return extendedVocabulary;
+    }
+
+    public Map<Integer, T> getIdxMap() {
+        return idxMap;
+    }
+
+    public AtomicLong getDocumentsCounter() {
+        return documentsCounter;
+    }
+
+    public int getMinWordFrequency() {
+        return minWordFrequency;
+    }
+
+    public void setMinWordFrequency(int minWordFrequency) {
+        this.minWordFrequency = minWordFrequency;
+    }
+
+    public boolean isHugeModelExpected() {
+        return hugeModelExpected;
+    }
+
+    public void setHugeModelExpected(boolean hugeModelExpected) {
+        this.hugeModelExpected = hugeModelExpected;
+    }
+
+    public List<String> getStopWords() {
+        return stopWords;
+    }
+
+    public void setStopWords(List<String> stopWords) {
+        this.stopWords = stopWords;
+    }
+
+    public int getScavengerThreshold() {
+        return scavengerThreshold;
+    }
+
+    public void setScavengerThreshold(int scavengerThreshold) {
+        this.scavengerThreshold = scavengerThreshold;
+    }
+
+    public int getRetentionDelay() {
+        return retentionDelay;
+    }
+
+    public void setRetentionDelay(int retentionDelay) {
+        this.retentionDelay = retentionDelay;
+    }
+
+    public AtomicLong getHiddenWordsCounter() {
+        return hiddenWordsCounter;
+    }
+
+    public void setHiddenWordsCounter(AtomicLong hiddenWordsCounter) {
+        this.hiddenWordsCounter = hiddenWordsCounter;
+    }
+
+    public AtomicLong getTotalWordCount() {
+        return totalWordCount;
+    }
+
+    public static ObjectMapper getMapper() {
+        return mapper;
+    }
+
+    public static void setMapper(ObjectMapper mapper) {
+        AbstractCache.mapper = mapper;
+    }
+}

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/InMemoryLookupCache.java

@@ -0,0 +1,480 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.models.word2vec.wordstore.inmemory;
+
+import org.deeplearning4j.models.word2vec.VocabWord;
+import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
+import org.deeplearning4j.text.movingwindow.Util;
+import org.nd4j.common.primitives.Counter;
+import org.nd4j.common.util.SerializationUtils;
+import org.nd4j.common.util.Index;
+
+import java.io.File;
+import java.io.InputStream;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Deprecated
+public class InMemoryLookupCache implements VocabCache<VocabWord>, Serializable {
+
+    private Index wordIndex = new Index();
+    public Counter<String> wordFrequencies = Util.parallelCounter();
+    public Counter<String> docFrequencies = Util.parallelCounter();
+    public Map<String, VocabWord> vocabs = new ConcurrentHashMap<>();
+    public Map<String, VocabWord> tokens = new ConcurrentHashMap<>();
+    private final AtomicLong totalWordOccurrences = new AtomicLong(0);
+    private int numDocs = 0;
+
+    public synchronized void setWordFrequencies(Counter<String> cnt) {
+        this.wordFrequencies = cnt;
+    }
+
+    public synchronized Map<String, VocabWord> getVocabs() {
+        return this.vocabs;
+    }
+
+    public synchronized void setVocabs(Map<String, VocabWord> vocabs) {
+        this.vocabs = vocabs;
+    }
+
+    public synchronized Counter<String> getWordFrequencies() {
+        return this.wordFrequencies;
+    }
+
+    public synchronized void setTokens(Map<String, VocabWord> tokens) {
+        this.tokens = tokens;
+    }
+
+    public synchronized Map<String, VocabWord> getTokens() {
+        return this.tokens;
+    }
+
+    public InMemoryLookupCache() {
+        //  this(false);
+    }
+
+    @Deprecated
+    public InMemoryLookupCache(boolean addUnk) {
+        /*if(addUnk) {
+            T word = (T) new SequenceElement(); //VocabWord(1.0, Word2Vec.UNK);
+            word.setIndex(0);
+            addToken(word);
+            addWordToIndex(0, Word2Vec.UNK);
+            putVocabWord(Word2Vec.UNK);
+        }
+        */
+    }
+
+    /**
+     * Returns all of the words in the vocab
+     *
+     * @returns all the words in the vocab
+     */
+    @Override
+    public synchronized Collection<String> words() {
+        return vocabs.keySet();
+    }
+
+
+    /**
+     * Increment the count for the given word
+     *
+     * @param word the word to increment the count for
+     */
+    @Override
+    public synchronized void incrementWordCount(String word) {
+        incrementWordCount(word, 1);
+    }
+
+    /**
+     * Increment the count for the given word by
+     * the amount increment
+     *
+     * @param word      the word to increment the count for
+     * @param increment the amount to increment by
+     */
+    @Override
+    public synchronized void incrementWordCount(String word, int increment) {
+        if (word == null || word.isEmpty())
+            throw new IllegalArgumentException("Word can't be empty or null");
+        wordFrequencies.incrementCount(word, increment);
+
+        if (hasToken(word)) {
+            VocabWord token = tokenFor(word);
+            token.increaseElementFrequency(increment);
+        }
+        totalWordOccurrences.set(totalWordOccurrences.get() + increment);
+    }
+
+    /**
+     * Returns the number of times the word has occurred
+     *
+     * @param word the word to retrieve the occurrence frequency for
+     * @return 0 if hasn't occurred or the number of times
+     * the word occurs
+     */
+    @Override
+    public synchronized int wordFrequency(String word) {
+        return (int) wordFrequencies.getCount(word);
+    }
+
+    /**
+     * Returns true if the cache contains the given word
+     *
+     * @param word the word to check for
+     * @return
+     */
+    @Override
+    public synchronized boolean containsWord(String word) {
+        return vocabs.containsKey(word);
+    }
+
+    /**
+     * Returns the word contained at the given index or null
+     *
+     * @param index the index of the word to get
+     * @return the word at the given index
+     */
+    @Override
+    public synchronized String wordAtIndex(int index) {
+        return (String) wordIndex.get(index);
+    }
+
+    @Override
+    public VocabWord elementAtIndex(int index) {
+        return wordFor(wordAtIndex(index));
+    }
+
+    /**
+     * Returns the index of a given word
+     *
+     * @param word the index of a given word
+     * @return the index of a given word or -1
+     * if not found
+     */
+    @Override
+    public synchronized int indexOf(String word) {
+        if (containsWord(word)) {
+            return wordFor(word).getIndex();
+        }
+        return -1;
+    }
+
+
+    /**
+     * Returns all of the vocab word nodes
+     *
+     * @return
+     */
+    @Override
+    public synchronized Collection<VocabWord> vocabWords() {
+        return vocabs.values();
+    }
+
+    /**
+     * The total number of word occurrences
+     *
+     * @return the total number of word occurrences
+     */
+    @Override
+    public synchronized long totalWordOccurrences() {
+        return totalWordOccurrences.get();
+    }
+
+
+
+    /**
+     * @param word
+     * @return
+     */
+    @Override
+    public synchronized VocabWord wordFor(String word) {
+        if (word == null)
+            return null;
+        VocabWord ret = vocabs.get(word);
+        return ret;
+    }
+
+    @Override
+    public VocabWord wordFor(long id) {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * @param index
+     * @param word
+     */
+    @Override
+    public synchronized void addWordToIndex(int index, String word) {
+        if (word == null || word.isEmpty())
+            throw new IllegalArgumentException("Word can't be empty or null");
+
+
+
+        if (!tokens.containsKey(word)) {
+            VocabWord token = new VocabWord(1.0, word);
+            tokens.put(word, token);
+            wordFrequencies.incrementCount(word, (float) 1.0);
+        }
+
+        /*
+            If we're speaking about adding any word to index directly, it means it's going to be vocab word, not token
+         */
+        if (!vocabs.containsKey(word)) {
+            VocabWord vw = tokenFor(word);
+            vw.setIndex(index);
+            vocabs.put(word, vw);
+            vw.setIndex(index);
+        }
+
+        if (!wordFrequencies.containsElement(word))
+            wordFrequencies.incrementCount(word, 1);
+
+        wordIndex.add(word, index);
+
+    }
+
+    @Override
+    public void addWordToIndex(int index, long elementId) {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * @param word
+     */
+    @Override
+    @Deprecated
+    public synchronized void putVocabWord(String word) {
+        if (word == null || word.isEmpty())
+            throw new IllegalArgumentException("Word can't be empty or null");
+        // STOP and UNK are not added as tokens
+        if (word.equals("STOP") || word.equals("UNK"))
+            return;
+        VocabWord token = tokenFor(word);
+        if (token == null)
+            throw new IllegalStateException("Word " + word + " not found as token in vocab");
+        int ind = token.getIndex();
+        addWordToIndex(ind, word);
+        if (!hasToken(word))
+            throw new IllegalStateException("Unable to add token " + word + " when not already a token");
+        vocabs.put(word, token);
+        wordIndex.add(word, token.getIndex());
+    }
+
+
+    /**
+     * Returns the number of words in the cache
+     *
+     * @return the number of words in the cache
+     */
+    @Override
+    public synchronized int numWords() {
+        return vocabs.size();
+    }
+
+    @Override
+    public synchronized int docAppearedIn(String word) {
+        return (int) docFrequencies.getCount(word);
+    }
+
+    @Override
+    public synchronized void incrementDocCount(String word, long howMuch) {
+        docFrequencies.incrementCount(word, howMuch);
+    }
+
+    @Override
+    public synchronized void setCountForDoc(String word, long count) {
+        docFrequencies.setCount(word, count);
+    }
+
+    @Override
+    public synchronized long totalNumberOfDocs() {
+        return numDocs;
+    }
+
+    @Override
+    public synchronized void incrementTotalDocCount() {
+        numDocs++;
+    }
+
+    @Override
+    public synchronized void incrementTotalDocCount(long by) {
+        numDocs += by;
+    }
+
+    @Override
+    public synchronized Collection<VocabWord> tokens() {
+        return tokens.values();
+    }
+
+    @Override
+    public synchronized boolean addToken(VocabWord word) {
+        if (null == tokens.put(word.getLabel(), word))
+            return true;
+        return false;
+    }
+
+    @Override
+    public synchronized VocabWord tokenFor(String word) {
+        return tokens.get(word);
+    }
+
+    @Override
+    public VocabWord tokenFor(long id) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public synchronized boolean hasToken(String token) {
+        return tokenFor(token) != null;
+    }
+
+    @Override
+    public void importVocabulary(VocabCache<VocabWord> vocabCache) {
+        for (VocabWord word : vocabCache.vocabWords()) {
+            if (vocabs.containsKey(word.getLabel())) {
+                wordFrequencies.incrementCount(word.getLabel(), (float) word.getElementFrequency());
+            } else {
+                tokens.put(word.getLabel(), word);
+                vocabs.put(word.getLabel(), word);
+                wordFrequencies.incrementCount(word.getLabel(), (float) word.getElementFrequency());
+            }
+            totalWordOccurrences.addAndGet((long) word.getElementFrequency());
+        }
+    }
+
+    @Override
+    public void updateWordsOccurrences() {
+        totalWordOccurrences.set(0);
+        for (VocabWord word : vocabWords()) {
+            totalWordOccurrences.addAndGet((long) word.getElementFrequency());
+        }
+    }
+
+    @Override
+    public void removeElement(String label) {
+        vocabs.remove(label);
+        tokens.remove(label);
+    }
+
+    @Override
+    public void removeElement(VocabWord element) {
+        removeElement(element.getLabel());
+    }
+
+
+    @Override
+    public synchronized void saveVocab() {
+        SerializationUtils.saveObject(this, new File("ser"));
+    }
+
+    @Override
+    public synchronized boolean vocabExists() {
+        return new File("ser").exists();
+    }
+
+
+    /**
+     * Load a look up cache from an input stream
+     * delimited by \n
+     * @param from the input stream to read from
+     * @return the in memory lookup cache
+     */
+    public static InMemoryLookupCache load(InputStream from) {
+        /*
+        Reader inputStream = new InputStreamReader(from);
+        LineIterator iter = IOUtils.lineIterator(inputStream);
+        String line;
+        InMemoryLookupCache ret = new InMemoryLookupCache();
+        int count = 0;
+        while((iter.hasNext())) {
+            line = iter.nextLine();
+            if(line.isEmpty())
+                continue;
+            ret.incrementWordCount(line);
+            VocabWord word = new VocabWord(1.0,line);
+            word.setIndex(count);
+            ret.addToken((SequenceElement) word);
+            ret.addWordToIndex(count,line);
+            ret.putVocabWord(line);
+            count++;
+        
+        }
+        
+        return ret; */
+        return null;
+    }
+
+    @Override
+    public synchronized void loadVocab() {
+        InMemoryLookupCache cache = SerializationUtils.readObject(new File("ser"));
+        this.vocabs = cache.vocabs;
+        this.wordFrequencies = cache.wordFrequencies;
+        this.wordIndex = cache.wordIndex;
+        this.tokens = cache.tokens;
+
+
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        InMemoryLookupCache that = (InMemoryLookupCache) o;
+
+        if (numDocs != that.numDocs)
+            return false;
+        if (wordIndex != null ? !wordIndex.equals(that.wordIndex) : that.wordIndex != null)
+            return false;
+        if (wordFrequencies != null ? !wordFrequencies.equals(that.wordFrequencies) : that.wordFrequencies != null)
+            return false;
+        if (docFrequencies != null ? !docFrequencies.equals(that.docFrequencies) : that.docFrequencies != null)
+            return false;
+        if (vocabWords().equals(that.vocabWords()))
+            return true;
+
+        return true;
+
+    }
+
+    @Override
+    public int hashCode() {
+        int result = wordIndex != null ? wordIndex.hashCode() : 0;
+        result = 31 * result + (wordFrequencies != null ? wordFrequencies.hashCode() : 0);
+        result = 31 * result + (docFrequencies != null ? docFrequencies.hashCode() : 0);
+        result = 31 * result + (vocabs != null ? vocabs.hashCode() : 0);
+        result = 31 * result + (tokens != null ? tokens.hashCode() : 0);
+        result = 31 * result + (totalWordOccurrences != null ? totalWordOccurrences.hashCode() : 0);
+        result = 31 * result + numDocs;
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "InMemoryLookupCache{" + "totalWordOccurrences=" + totalWordOccurrences + ", numDocs=" + numDocs + '}';
+    }
+}