chore: import upstream snapshot with attribution

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/pom.xml

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ /* ******************************************************************************
+  ~  *
+  ~  *
+  ~  * This program and the accompanying materials are made available under the
+  ~  * terms of the Apache License, Version 2.0 which is available at
+  ~  * https://www.apache.org/licenses/LICENSE-2.0.
+  ~  *
+  ~  *  See the NOTICE file distributed with this work for additional
+  ~  *  information regarding copyright ownership.
+  ~  * Unless required by applicable law or agreed to in writing, software
+  ~  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  ~  * License for the specific language governing permissions and limitations
+  ~  * under the License.
+  ~  *
+  ~  * SPDX-License-Identifier: Apache-2.0
+  ~  ******************************************************************************/
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.eclipse.deeplearning4j</groupId>
+        <artifactId>deeplearning4j-data</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>deeplearning4j-datasets</artifactId>
+    <packaging>jar</packaging>
+
+    <name>deeplearning4j-datasets</name>
+
+    <properties>
+        <module.name>deeplearning4j.datasets</module.name>
+    </properties>
+
+    <build>
+        <plugins>
+           <plugin>
+                <groupId>org.moditect</groupId>
+                <artifactId>moditect-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>datavec-data-image</artifactId>
+            <version>${datavec.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>deeplearning4j-datavec-iterators</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>resources</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+
+
+</project>

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/base/EmnistFetcher.java

@@ -0,0 +1,127 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.base;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.eclipse.deeplearning4j.resources.DataSetResource;
+import org.eclipse.deeplearning4j.resources.ResourceDataSets;
+import org.eclipse.deeplearning4j.resources.utils.EMnistSet;
+
+import java.io.File;
+import java.io.IOException;
+
+@Slf4j
+public class EmnistFetcher extends MnistFetcher {
+
+    private final EMnistSet ds;
+    @Getter
+    private DataSetResource emnistDataTrain;
+    @Getter
+    private DataSetResource emnistDataTest;
+    @Getter
+    private DataSetResource emnistLabelsTrain;
+    @Getter
+    private DataSetResource emnistLabelsTest;
+    @Getter
+    private DataSetResource emnistMappingTrain;
+    @Getter
+    private DataSetResource emnistMappingTest;
+
+    public EmnistFetcher() {
+        this(EMnistSet.MNIST);
+    }
+
+    public EmnistFetcher(EMnistSet ds) {
+        this.ds = ds;
+        emnistDataTrain = ResourceDataSets.emnistTrain(ds);
+        emnistDataTest = ResourceDataSets.emnistTest(ds);
+        emnistLabelsTrain = ResourceDataSets.emnistLabelsTrain(ds);
+        emnistLabelsTest = ResourceDataSets.emnistLabelsTest(ds);
+        emnistMappingTrain = ResourceDataSets.emnistMappingTrain(ds);
+        emnistMappingTest = ResourceDataSets.emnistMappingTest(ds);
+
+    }
+
+    public EmnistFetcher(EMnistSet ds,File topLevelDir) {
+        this.ds = ds;
+        emnistDataTrain = ResourceDataSets.emnistTrain(ds,topLevelDir);
+        emnistDataTest = ResourceDataSets.emnistTest(ds,topLevelDir);
+        emnistLabelsTrain = ResourceDataSets.emnistLabelsTrain(ds,topLevelDir);
+        emnistLabelsTest = ResourceDataSets.emnistLabelsTest(ds,topLevelDir);
+        emnistMappingTrain = ResourceDataSets.emnistMappingTrain(ds,topLevelDir);
+        emnistMappingTest = ResourceDataSets.emnistMappingTest(ds,topLevelDir);
+
+    }
+
+
+    @Override
+    public String getName() {
+        return "EMNIST";
+    }
+
+    // --- Train files ---
+
+    public static int numLabels(EMnistSet dataSet) {
+        switch (dataSet) {
+            case COMPLETE:
+                return 62;
+            case MERGE:
+                return 47;
+            case BALANCED:
+                return 47;
+            case LETTERS:
+                return 26;
+            case DIGITS:
+                return 10;
+            case MNIST:
+                return 10;
+            default:
+                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
+        }
+    }
+
+    @Override
+    public File downloadAndUntar() throws IOException {
+        if (fileDir != null) {
+            return fileDir;
+        }
+
+        File baseDir = getBaseDir();
+        if (!(baseDir.isDirectory() || baseDir.mkdir())) {
+            throw new IOException("Could not mkdir " + baseDir);
+        }
+
+        log.info("Downloading {}...", getName());
+        // get features
+        emnistDataTrain.download(true,3,300000,30000);
+        emnistDataTest.download(true,3,300000,30000);
+        emnistLabelsTrain.download(true,3,300000,30000);
+        emnistLabelsTest.download(true,3,300000,30000);
+        emnistMappingTrain.download(false,3,300000,30000);
+        emnistMappingTest.download(false,3,300000,30000);
+
+
+        // get labels
+        fileDir = baseDir;
+        return fileDir;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/base/IrisUtils.java

@@ -0,0 +1,90 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.base;
+
+import org.apache.commons.io.IOUtils;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.common.resources.Downloader;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+public class IrisUtils {
+
+    private static final String IRIS_RELATIVE_URL = "datasets/iris.dat";
+    private static final String MD5 = "1c21400a78061197eac64c6748844216";
+
+    private IrisUtils() {}
+
+    public static List<DataSet> loadIris(int from, int to) throws IOException {
+        File rootDir = DL4JResources.getDirectory(ResourceType.DATASET, "iris");
+        File irisData = new File(rootDir, "iris.dat");
+        if(!irisData.exists()) {
+            URL url = DL4JResources.getURL(IRIS_RELATIVE_URL);
+            Downloader.download("Iris", url, irisData, MD5, 3);
+        }
+
+        @SuppressWarnings("unchecked")
+        List<String> lines;
+        try(InputStream is = new FileInputStream(irisData)){
+            lines = IOUtils.readLines(is);
+        }
+        List<DataSet> list = new ArrayList<>();
+        INDArray ret = to - from > 1 ? Nd4j.ones(Math.abs(to - from), 4) : Nd4j.ones( 4);
+        double[][] outcomes = new double[lines.size()][3];
+        int putCount = 0;
+
+        for (int i = from; i < to; i++) {
+            String line = lines.get(i);
+            String[] split = line.split(",");
+
+            addRow(ret, putCount++, split);
+
+            String outcome = split[split.length - 1];
+            double[] rowOutcome = new double[3];
+            rowOutcome[Integer.parseInt(outcome)] = 1;
+            outcomes[i] = rowOutcome;
+        }
+
+        for (int i = 0; i < ret.rows(); i++) {
+            DataSet add = new DataSet(ret.getRow(i, false), Nd4j.create(outcomes[from + i], 3));
+            list.add(add);
+        }
+        return list;
+    }
+
+    private static void addRow(INDArray ret, int row, String[] line) {
+        double[] vector = new double[4];
+        for (int i = 0; i < 4; i++)
+            vector[i] = Double.parseDouble(line[i]);
+
+        ret.putRow(row, Nd4j.create(vector));
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/base/MnistFetcher.java

@@ -0,0 +1,89 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.base;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.eclipse.deeplearning4j.resources.DataSetResource;
+import org.eclipse.deeplearning4j.resources.ResourceDataSets;
+import org.nd4j.common.resources.Downloader;
+
+import java.io.*;
+import java.net.URL;
+
+@Data
+@NoArgsConstructor
+@Slf4j
+public class MnistFetcher {
+
+    protected static final String LOCAL_DIR_NAME = "MNIST";
+
+    protected File fileDir;
+
+    private DataSetResource mnistTrain = ResourceDataSets.mnistTrain();
+    private DataSetResource mnistTest = ResourceDataSets.mnistTest();
+    private DataSetResource mnistTrainLabels = ResourceDataSets.mnistTrainLabels();
+    private DataSetResource mnistTestLabels = ResourceDataSets.mnistTestLabels();
+
+
+    public MnistFetcher(File tempDir) {
+        this.fileDir = tempDir;
+    }
+
+
+    public String getName() {
+        return "MNIST";
+    }
+
+    public File getBaseDir() {
+        return DL4JResources.getDirectory(ResourceType.DATASET, getName());
+    }
+
+
+
+    public File downloadAndUntar() throws IOException {
+        if (fileDir != null) {
+            return fileDir;
+        }
+
+        File baseDir = getBaseDir();
+        if (!(baseDir.isDirectory() || baseDir.mkdir())) {
+            throw new IOException("Could not mkdir " + baseDir);
+        }
+
+        log.info("Downloading {}...", getName());
+        // get features
+
+
+        mnistTrain.download(true,3,200000,20000);
+        mnistTest.download(true,3,200000,20000);
+
+        mnistTrainLabels.download(true,3,200000,20000);
+        mnistTestLabels.download(true,3,200000,20000);
+
+        // get labels
+        fileDir = baseDir;
+        return fileDir;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/CacheableDataSet.java

@@ -0,0 +1,37 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.image.transform.ImageTransform;
+
+interface CacheableDataSet {
+
+    String remoteDataUrl();
+    String remoteDataUrl(DataSetType set);
+    String localCacheName();
+    String dataSetName(DataSetType set);
+    long expectedChecksum();
+    long expectedChecksum(DataSetType set);
+    boolean isCached();
+    RecordReader getRecordReader(long rngSeed, int[] imgDim, DataSetType set, ImageTransform imageTransform);
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/CacheableExtractableDataSetFetcher.java

@@ -0,0 +1,125 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.nd4j.common.util.ArchiveUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.zip.Adler32;
+import java.util.zip.Checksum;
+
+@Slf4j
+public abstract class CacheableExtractableDataSetFetcher implements CacheableDataSet {
+
+    @Override public String dataSetName(DataSetType set) { return ""; }
+    @Override public String remoteDataUrl() { return remoteDataUrl(DataSetType.TRAIN); }
+    @Override public long expectedChecksum() { return expectedChecksum(DataSetType.TRAIN); }
+    public void downloadAndExtract() throws IOException { downloadAndExtract(DataSetType.TRAIN); }
+
+    /**
+     * Downloads and extracts the local dataset.
+     *
+     * @throws IOException
+     */
+    public void downloadAndExtract(DataSetType set) throws IOException {
+        String localFilename = new File(remoteDataUrl(set)).getName();
+        File tmpFile = new File(System.getProperty("java.io.tmpdir"), localFilename);
+        File localCacheDir = getLocalCacheDir();
+
+        // check empty cache
+        if(localCacheDir.exists()) {
+            File[] list = localCacheDir.listFiles();
+            if(list == null || list.length == 0)
+                localCacheDir.delete();
+        }
+
+        File localDestinationDir = new File(localCacheDir, dataSetName(set));
+        if(!localDestinationDir.exists()) {
+            localCacheDir.mkdirs();
+            tmpFile.delete();
+            log.info("Downloading dataset to " + tmpFile.getAbsolutePath());
+            FileUtils.copyURLToFile(new URL(remoteDataUrl(set)), tmpFile);
+        } else {
+            //Directory exists and is non-empty - assume OK
+            log.info("Using cached dataset at " + localCacheDir.getAbsolutePath());
+            return;
+        }
+
+        if(expectedChecksum(set) != 0L) {
+            log.info("Verifying download...");
+            Checksum adler = new Adler32();
+            FileUtils.checksum(tmpFile, adler);
+            long localChecksum = adler.getValue();
+            log.info("Checksum local is " + localChecksum + ", expecting "+expectedChecksum(set));
+
+            if(expectedChecksum(set) != localChecksum) {
+                log.error("Checksums do not match. Cleaning up files and failing...");
+                tmpFile.delete();
+                throw new IllegalStateException( "Dataset file failed checksum: " + tmpFile + " - expected checksum " + expectedChecksum(set)
+                + " vs. actual checksum " + localChecksum + ". If this error persists, please open an issue at https://github.com/eclipse/deeplearning4j.");
+            }
+        }
+
+        try {
+            ArchiveUtils.unzipFileTo(tmpFile.getAbsolutePath(), localCacheDir.getAbsolutePath(), false);
+        } catch (Throwable t){
+            //Catch any errors during extraction, and delete the directory to avoid leaving the dir in an invalid state
+            if(localCacheDir.exists())
+                FileUtils.deleteDirectory(localCacheDir);
+            throw t;
+        }
+    }
+
+    protected File getLocalCacheDir(){
+        return DL4JResources.getDirectory(ResourceType.DATASET, localCacheName());
+    }
+
+    /**
+     * Returns a boolean indicating if the dataset is already cached locally.
+     *
+     * @return boolean
+     */
+    @Override
+    public boolean isCached() {
+        return getLocalCacheDir().exists();
+    }
+
+
+    protected static void deleteIfEmpty(File localCache){
+        if(localCache.exists()) {
+            File[] files = localCache.listFiles();
+            if(files == null || files.length < 1){
+                try {
+                    FileUtils.deleteDirectory(localCache);
+                } catch (IOException e){
+                    //Ignore
+                    log.debug("Error deleting directory: {}", localCache);
+                }
+            }
+        }
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/Cifar10Fetcher.java

@@ -0,0 +1,104 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import org.datavec.api.io.filters.RandomPathFilter;
+import org.datavec.api.io.labels.ParentPathLabelGenerator;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.split.FileSplit;
+import org.datavec.api.split.InputSplit;
+import org.datavec.image.loader.BaseImageLoader;
+import org.datavec.image.recordreader.ImageRecordReader;
+import org.datavec.image.transform.ImageTransform;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.nd4j.common.base.Preconditions;
+
+import java.io.File;
+import java.util.Random;
+
+public class Cifar10Fetcher extends CacheableExtractableDataSetFetcher {
+    public static final String LABELS_FILENAME = "labels.txt";
+    public static final String LOCAL_CACHE_NAME = "cifar10";
+
+    public static int INPUT_WIDTH = 32;
+    public static int INPUT_HEIGHT = 32;
+    public static int INPUT_CHANNELS = 3;
+    public static int NUM_LABELS = 10;
+
+    @Override
+    public String remoteDataUrl(DataSetType set) {
+        return DL4JResources.getURLString("datasets/cifar10_dl4j.v1.zip");
+    }
+    @Override
+    public String localCacheName(){ return LOCAL_CACHE_NAME; }
+    @Override
+    public long expectedChecksum(DataSetType set) { return 292852033L; }
+    @Override
+    public RecordReader getRecordReader(long rngSeed, int[] imgDim, DataSetType set, ImageTransform imageTransform) {
+        Preconditions.checkState(imgDim == null || imgDim.length == 2, "Invalid image dimensions: must be null or lenth 2. Got: %s", imgDim);
+        // check empty cache
+        File localCache = getLocalCacheDir();
+        deleteIfEmpty(localCache);
+
+        try {
+            if (!localCache.exists()){
+                downloadAndExtract();
+            }
+        } catch(Exception e) {
+            throw new RuntimeException("Could not download CIFAR-10", e);
+        }
+
+        Random rng = new Random(rngSeed);
+        File datasetPath;
+        switch (set) {
+            case TRAIN:
+                datasetPath = new File(localCache, "/train/");
+                break;
+            case TEST:
+                datasetPath = new File(localCache, "/test/");
+                break;
+            case VALIDATION:
+                throw new IllegalArgumentException("You will need to manually create and iterate a validation directory, CIFAR-10 does not provide labels");
+
+            default:
+                datasetPath = new File(localCache, "/train/");
+        }
+
+        // set up file paths
+        RandomPathFilter pathFilter = new RandomPathFilter(rng, BaseImageLoader.ALLOWED_FORMATS);
+        FileSplit filesInDir = new FileSplit(datasetPath, BaseImageLoader.ALLOWED_FORMATS, rng);
+        InputSplit[] filesInDirSplit = filesInDir.sample(pathFilter, 1);
+
+        int h = (imgDim == null ? Cifar10Fetcher.INPUT_HEIGHT : imgDim[0]);
+        int w = (imgDim == null ? Cifar10Fetcher.INPUT_WIDTH : imgDim[1]);
+        ImageRecordReader rr = new ImageRecordReader(h, w, Cifar10Fetcher.INPUT_CHANNELS, new ParentPathLabelGenerator(), imageTransform);
+
+        try {
+            rr.initialize(filesInDirSplit[0]);
+        } catch(Exception e) {
+            throw new RuntimeException(e);
+        }
+
+        return rr;
+    }
+
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/DataSetType.java

@@ -0,0 +1,25 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+public enum DataSetType {
+    TRAIN, TEST, VALIDATION
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/EmnistDataFetcher.java

@@ -0,0 +1,115 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.deeplearning4j.datasets.base.EmnistFetcher;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.eclipse.deeplearning4j.resources.utils.EMnistSet;
+import org.deeplearning4j.datasets.iterator.impl.EmnistDataSetIterator;
+import org.deeplearning4j.datasets.mnist.MnistManager;
+import org.nd4j.linalg.dataset.api.iterator.fetcher.DataSetFetcher;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+
+
+@Slf4j
+public class EmnistDataFetcher extends MnistDataFetcher implements DataSetFetcher {
+
+    protected EmnistFetcher fetcher;
+
+
+
+    public EmnistDataFetcher(EMnistSet dataSet, boolean binarize, boolean train, boolean shuffle,
+                             long rngSeed,File topLevelDir) throws IOException {
+        fetcher = new EmnistFetcher(dataSet,topLevelDir);
+        if (!emnistExists(fetcher)) {
+            fetcher.downloadAndUntar();
+        }
+
+
+        String EMNIST_ROOT = topLevelDir.getAbsolutePath();
+        if (train) {
+            images = fetcher.getEmnistDataTrain().localPath().getAbsolutePath();
+            labels = fetcher.getEmnistLabelsTrain().localPath().getAbsolutePath();
+            totalExamples = EmnistDataSetIterator.numExamplesTrain(dataSet);
+        } else {
+            images = fetcher.getEmnistDataTest().localPath().getAbsolutePath();
+            labels = fetcher.getEmnistLabelsTest().localPath().getAbsolutePath();
+            totalExamples = EmnistDataSetIterator.numExamplesTest(dataSet);
+        }
+        try {
+            manager = new MnistManager(images, labels, totalExamples);
+        } catch (Exception e) {
+            log.error("",e);
+            FileUtils.deleteDirectory(new File(EMNIST_ROOT));
+            new EmnistFetcher(dataSet).downloadAndUntar();
+            manager = new MnistManager(images, labels, totalExamples);
+        }
+
+        numOutcomes = EmnistDataSetIterator.numLabels(dataSet);
+        this.binarize = binarize;
+        cursor = 0;
+        manager.setCurrent(cursor);
+        inputColumns = manager.getImages().getEntryLength();
+        this.train = train;
+        this.shuffle = shuffle;
+
+        order = new int[totalExamples];
+        for (int i = 0; i < order.length; i++)
+            order[i] = i;
+        rng = new Random(rngSeed);
+        reset(); //Shuffle order
+
+
+        //For some inexplicable reason, EMNIST LETTERS set is indexed 1 to 26 (i.e., 1 to nClasses), while everything else
+        // is indexed (0 to nClasses-1) :/
+        if (dataSet == EMnistSet.LETTERS) {
+            oneIndexed = true;
+        } else {
+            oneIndexed = false;
+        }
+        this.fOrder = true; //MNIST is C order, EMNIST is F order
+    }
+
+    public EmnistDataFetcher(EMnistSet dataSet, boolean binarize, boolean train, boolean shuffle,
+                             long rngSeed) throws IOException {
+        this(dataSet,binarize,train,shuffle,rngSeed,DL4JResources.getDirectory(ResourceType.DATASET, "EMNIST"));
+    }
+
+    private boolean emnistExists(EmnistFetcher e) {
+        //Check 4 files:
+        if (!fetcher.getEmnistDataTrain().existsLocally())
+            return false;
+        if (!fetcher.getEmnistLabelsTrain().existsLocally())
+            return false;
+        if (!fetcher.getEmnistDataTest().existsLocally())
+            return false;
+        if (!fetcher.getEmnistLabelsTest().existsLocally())
+            return false;
+        return true;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/IrisDataFetcher.java

@@ -0,0 +1,60 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import org.deeplearning4j.datasets.base.IrisUtils;
+import org.nd4j.linalg.dataset.api.iterator.fetcher.BaseDataFetcher;
+
+import java.io.IOException;
+
+
+public class IrisDataFetcher extends BaseDataFetcher {
+
+
+    /**
+     * 
+     */
+    private static final long serialVersionUID = 4566329799221375262L;
+    public final static int NUM_EXAMPLES = 150;
+
+    public IrisDataFetcher() {
+        numOutcomes = 3;
+        inputColumns = 4;
+        totalExamples = NUM_EXAMPLES;
+    }
+
+    @Override
+    public void fetch(int numExamples) {
+        int from = cursor;
+        int to = cursor + numExamples;
+        if (to > totalExamples)
+            to = totalExamples;
+
+        try {
+            initializeCurrFromList(IrisUtils.loadIris(from, to));
+            cursor += numExamples;
+        } catch (IOException e) {
+            throw new IllegalStateException("Unable to load iris.dat", e);
+        }
+    }
+
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/MnistDataFetcher.java

@@ -0,0 +1,278 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import lombok.SneakyThrows;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.deeplearning4j.datasets.base.MnistFetcher;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.deeplearning4j.datasets.mnist.MnistManager;
+import org.eclipse.deeplearning4j.resources.DataSetResource;
+import org.eclipse.deeplearning4j.resources.ResourceDataSets;
+import org.nd4j.linalg.api.buffer.DataType;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.iterator.fetcher.BaseDataFetcher;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.indexing.NDArrayIndex;
+import org.nd4j.common.util.MathUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.zip.Adler32;
+import java.util.zip.Checksum;
+
+
+public class MnistDataFetcher extends BaseDataFetcher {
+    public static final int NUM_EXAMPLES = 60000;
+    public static final int NUM_EXAMPLES_TEST = 10000;
+
+    protected static final long CHECKSUM_TRAIN_FEATURES = 2094436111L;
+    protected static final long CHECKSUM_TRAIN_LABELS = 4008842612L;
+    protected static final long CHECKSUM_TEST_FEATURES = 2165396896L;
+    protected static final long CHECKSUM_TEST_LABELS = 2212998611L;
+
+    protected static final long[] CHECKSUMS_TRAIN = new long[]{CHECKSUM_TRAIN_FEATURES, CHECKSUM_TRAIN_LABELS};
+    protected static final long[] CHECKSUMS_TEST = new long[]{CHECKSUM_TEST_FEATURES, CHECKSUM_TEST_LABELS};
+
+    protected boolean binarize = true;
+    protected boolean train;
+    protected int[] order;
+    protected Random rng;
+    protected boolean shuffle;
+    protected boolean oneIndexed = false;
+    protected boolean fOrder = false; //MNIST is C order, EMNIST is F order
+
+    protected boolean firstShuffle = true;
+    protected  int numExamples = 0;
+    protected String images,labels;
+    //note: we default to zero here on purpose, otherwise when first initializes an error is thrown.
+    private long lastCursor = 0;
+    protected MnistManager manager;
+
+    /**
+     * Constructor telling whether to binarize the dataset or not
+     * @param binarize whether to binarize the dataset or not
+     * @throws IOException
+     */
+    public MnistDataFetcher(boolean binarize) throws IOException {
+        this(binarize, true, true, System.currentTimeMillis(), NUM_EXAMPLES);
+    }
+
+
+
+    public MnistDataFetcher(boolean binarize, boolean train, boolean shuffle, long rngSeed, int numExamples,File topLevelDir) throws IOException {
+        if(this instanceof EmnistDataFetcher)
+            return;
+
+
+        this.topLevelDir = topLevelDir;
+        long[] checksums;
+        DataSetResource imageResource = null;
+        DataSetResource labelResource = null;
+        if (train) {
+            imageResource = topLevelDir() != null ?  ResourceDataSets.mnistTrain(topLevelDir()) :  ResourceDataSets.mnistTrain();
+            if(!imageResource.existsLocally())
+                imageResource.download(true,3,200000,20000);
+
+            labelResource = topLevelDir() != null ? ResourceDataSets.mnistTrainLabels(topLevelDir()) : ResourceDataSets.mnistTrainLabels();
+            if(!labelResource.existsLocally())
+                labelResource.download(true,3,200000,20000);
+
+            totalExamples = NUM_EXAMPLES;
+            checksums = CHECKSUMS_TRAIN;
+        } else {
+            imageResource = topLevelDir() != null ?  ResourceDataSets.mnistTest(topLevelDir()) :  ResourceDataSets.mnistTest();
+            if(!imageResource.existsLocally())
+                imageResource.download(true,3,200000,20000);
+
+            labelResource = topLevelDir() != null ? ResourceDataSets.mnistTestLabels(topLevelDir()) : ResourceDataSets.mnistTestLabels();
+            if(!labelResource.existsLocally())
+                labelResource.download(true,3,200000,20000);
+
+            totalExamples = NUM_EXAMPLES_TEST;
+            checksums = CHECKSUMS_TEST;
+        }
+
+        images = imageResource.localPath().getAbsolutePath();
+        labels = labelResource.localPath().getAbsolutePath();
+
+
+        try {
+            manager = new MnistManager(images, labels, train);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+
+
+        numOutcomes = 10;
+        this.binarize = binarize;
+        cursor = 0;
+        inputColumns = manager.getImages().getEntryLength();
+        this.train = train;
+        this.shuffle = shuffle;
+
+        if (train) {
+            order = new int[NUM_EXAMPLES];
+        } else {
+            order = new int[NUM_EXAMPLES_TEST];
+        }
+        for (int i = 0; i < order.length; i++)
+            order[i] = i;
+        rng = new Random(rngSeed);
+        this.numExamples = numExamples;
+        reset(); //Shuffle order
+    }
+
+    public MnistDataFetcher(boolean binarize, boolean train, boolean shuffle, long rngSeed, int numExamples) throws IOException {
+        this(binarize,train,shuffle,rngSeed,numExamples,null);
+    }
+
+
+
+    private void validateFiles(String[] files, long[] checksums) {
+        //Validate files:
+        try {
+            for (int i = 0; i < files.length; i++) {
+                File f = new File(files[i]);
+                Checksum adler = new Adler32();
+                long checksum = f.exists() ? FileUtils.checksum(f, adler).getValue() : -1;
+                if (!f.exists() || checksum != checksums[i]) {
+                    throw new IllegalStateException("Failed checksum: expected " + checksums[i] +
+                            ", got " + checksum + " for file: " + f);
+                }
+            }
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public MnistDataFetcher() throws IOException {
+        this(true);
+    }
+
+    private float[][] featureData = null;
+
+    @SneakyThrows
+    @Override
+    public void fetch(int numExamples) {
+        if (!hasMore()) {
+            throw new IllegalStateException("Unable to get more; there are no more images");
+        }
+
+        manager.setCurrent((int) lastCursor);
+        INDArray labels = Nd4j.zeros(DataType.FLOAT, numExamples, numOutcomes);
+
+        if(featureData == null || featureData.length < numExamples){
+            featureData = new float[numExamples][28 * 28];
+        }
+
+        int actualExamples = 0;
+        byte[] working = null;
+        for (int i = 0; i < numExamples; i++, cursor++) {
+            if (!hasMore())
+                break;
+
+            manager.setCurrent(cursor);
+            lastCursor = cursor;
+            byte[] img = manager.readImageUnsafe(order[cursor]);
+
+            if (fOrder) {
+                //EMNIST requires F order to C order
+                if (working == null) {
+                    working = new byte[28 * 28];
+                }
+                for (int j = 0; j < 28 * 28; j++) {
+                    working[j] = img[28 * (j % 28) + j / 28];
+                }
+                img = working;
+            }
+
+            int label = manager.readLabel(order[cursor]);
+            if (oneIndexed) {
+                //For some inexplicable reason, Emnist LETTERS set is indexed 1 to 26 (i.e., 1 to nClasses), while everything else
+                // is indexed (0 to nClasses-1) :/
+                label--;
+            }
+
+            labels.put(actualExamples, label, 1.0f);
+
+            for(int j = 0 ; j < img.length ; j++) {
+                featureData[actualExamples][j] = ((int) img[j]) & 0xFF;
+            }
+
+            actualExamples++;
+        }
+
+        INDArray features;
+
+        if(featureData.length == actualExamples){
+            features = Nd4j.create(featureData);
+        } else {
+            features = Nd4j.create(Arrays.copyOfRange(featureData, 0, actualExamples));
+        }
+
+        if (actualExamples < numExamples) {
+            labels = labels.get(NDArrayIndex.interval(0, actualExamples), NDArrayIndex.all());
+        }
+
+        if(binarize){
+            features = features.gt(30.0).castTo(DataType.FLOAT);
+        } else {
+            features.divi(255.0);
+        }
+
+        curr = new DataSet(features, labels);
+    }
+
+    @Override
+    public void reset() {
+        cursor = 0;
+        curr = null;
+        if (shuffle) {
+            if((train && numExamples < NUM_EXAMPLES) || (!train && numExamples < NUM_EXAMPLES_TEST)){
+                //Shuffle only first N elements
+                if(firstShuffle){
+                    MathUtils.shuffleArray(order, rng);
+                    firstShuffle = false;
+                } else {
+                    MathUtils.shuffleArraySubset(order, numExamples, rng);
+                }
+            } else {
+                MathUtils.shuffleArray(order, rng);
+            }
+        }
+    }
+
+    @Override
+    public DataSet next() {
+        DataSet next = super.next();
+        return next;
+    }
+
+    public void close() {
+    }
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcher.java

@@ -0,0 +1,132 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.split.FileSplit;
+import org.datavec.image.loader.BaseImageLoader;
+import org.datavec.image.recordreader.objdetect.ObjectDetectionRecordReader;
+import org.datavec.image.transform.ImageTransform;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+
+public class SvhnDataFetcher extends CacheableExtractableDataSetFetcher {
+
+    private static String BASE_URL = "http://ufldl.stanford.edu/";
+
+    public static void setBaseUrl(String baseUrl){
+        BASE_URL = baseUrl;
+    }
+
+    public static int NUM_LABELS = 10;
+
+    @Override
+    public String remoteDataUrl(DataSetType set) {
+        switch (set) {
+            case TRAIN:
+                return BASE_URL + "housenumbers/train.tar.gz";
+            case TEST:
+                return BASE_URL + "housenumbers/test.tar.gz";
+            case VALIDATION:
+                return BASE_URL + "housenumbers/extra.tar.gz";
+            default:
+                 throw new IllegalArgumentException("Unknown DataSetType:" + set);
+        }
+    }
+
+    @Override
+    public String localCacheName() {
+        return "SVHN";
+    }
+
+    @Override
+    public String dataSetName(DataSetType set) {
+        switch (set) {
+            case TRAIN:
+                return "train";
+            case TEST:
+                return "test";
+            case VALIDATION:
+                return "extra";
+            default:
+                throw new IllegalArgumentException("Unknown DataSetType:" + set);
+        }
+    }
+
+    @Override
+    public long expectedChecksum(DataSetType set) {
+        switch (set) {
+            case TRAIN:
+                return 979655493L;
+            case TEST:
+                return 1629515343L;
+            case VALIDATION:
+                return 132781169L;
+            default:
+                 throw new IllegalArgumentException("Unknown DataSetType:" + set);
+        }
+    }
+
+    public File getDataSetPath(DataSetType set) throws IOException {
+        File localCache = getLocalCacheDir();
+        // check empty cache
+        deleteIfEmpty(localCache);
+
+        File datasetPath;
+        switch (set) {
+            case TRAIN:
+                datasetPath = new File(localCache, "/train/");
+                break;
+            case TEST:
+                datasetPath = new File(localCache, "/test/");
+                break;
+            case VALIDATION:
+                datasetPath = new File(localCache, "/extra/");
+                break;
+            default:
+                datasetPath = null;
+        }
+
+        if (!datasetPath.exists()) {
+            downloadAndExtract(set);
+        }
+        return datasetPath;
+    }
+
+    @Override
+    public RecordReader getRecordReader(long rngSeed, int[] imgDim, DataSetType set, ImageTransform imageTransform) {
+        try {
+            Random rng = new Random(rngSeed);
+            File datasetPath = getDataSetPath(set);
+
+            FileSplit data = new FileSplit(datasetPath, BaseImageLoader.ALLOWED_FORMATS, rng);
+            ObjectDetectionRecordReader recordReader = new ObjectDetectionRecordReader(imgDim[1], imgDim[0], imgDim[2],
+                            imgDim[4], imgDim[3], null);
+
+            recordReader.initialize(data);
+            return recordReader;
+        } catch (IOException e) {
+            throw new RuntimeException("Could not download SVHN", e);
+        }
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/TinyImageNetFetcher.java

@@ -0,0 +1,105 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import org.datavec.api.io.filters.RandomPathFilter;
+import org.datavec.api.io.labels.ParentPathLabelGenerator;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.split.FileSplit;
+import org.datavec.api.split.InputSplit;
+import org.datavec.image.loader.BaseImageLoader;
+import org.datavec.image.recordreader.ImageRecordReader;
+import org.datavec.image.transform.ImageTransform;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.nd4j.common.base.Preconditions;
+
+import java.io.File;
+import java.util.Random;
+
+public class TinyImageNetFetcher extends CacheableExtractableDataSetFetcher {
+    public static final String WORDS_FILENAME = "words.txt";
+    public static final String LOCAL_CACHE_NAME = "TINYIMAGENET_200";
+
+    public static int INPUT_WIDTH = 64;
+    public static int INPUT_HEIGHT = 64;
+    public static int INPUT_CHANNELS = 3;
+    public static int NUM_LABELS = 200;
+    public static int NUM_EXAMPLES = NUM_LABELS*500;
+
+    @Override
+    public String remoteDataUrl(DataSetType set) {
+        return DL4JResources.getURLString("datasets/tinyimagenet_200_dl4j.v1.zip");
+    }
+    @Override
+    public String localCacheName(){ return LOCAL_CACHE_NAME; }
+    @Override
+    public long expectedChecksum(DataSetType set) { return 33822361L; }
+    @Override
+    public RecordReader getRecordReader(long rngSeed, int[] imgDim, DataSetType set, ImageTransform imageTransform) {
+        Preconditions.checkState(imgDim == null || imgDim.length == 2, "Invalid image dimensions: must be null or lenth 2. Got: %s", imgDim);
+        // check empty cache
+        File localCache = getLocalCacheDir();
+        deleteIfEmpty(localCache);
+
+        try {
+            if (!localCache.exists()){
+                downloadAndExtract();
+            }
+        } catch(Exception e) {
+            throw new RuntimeException("Could not download TinyImageNet", e);
+        }
+
+        Random rng = new Random(rngSeed);
+        File datasetPath;
+        switch (set) {
+            case TRAIN:
+                datasetPath = new File(localCache, "/train/");
+                break;
+            case TEST:
+                datasetPath = new File(localCache, "/test/");
+                break;
+            case VALIDATION:
+                throw new IllegalArgumentException("You will need to manually iterate the /validation/images/ directory, TinyImageNet does not provide labels");
+
+            default:
+                datasetPath = new File(localCache, "/train/");
+        }
+
+        // set up file paths
+        RandomPathFilter pathFilter = new RandomPathFilter(rng, BaseImageLoader.ALLOWED_FORMATS);
+        FileSplit filesInDir = new FileSplit(datasetPath, BaseImageLoader.ALLOWED_FORMATS, rng);
+        InputSplit[] filesInDirSplit = filesInDir.sample(pathFilter, 1);
+
+        int h = (imgDim == null ? TinyImageNetFetcher.INPUT_HEIGHT : imgDim[0]);
+        int w = (imgDim == null ? TinyImageNetFetcher.INPUT_WIDTH : imgDim[1]);
+        ImageRecordReader rr = new ImageRecordReader(h, w,TinyImageNetFetcher.INPUT_CHANNELS, new ParentPathLabelGenerator(), imageTransform);
+
+        try {
+            rr.initialize(filesInDirSplit[0]);
+        } catch(Exception e) {
+            throw new RuntimeException(e);
+        }
+
+        return rr;
+    }
+
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/UciSequenceDataFetcher.java

@@ -0,0 +1,164 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader;
+import org.datavec.api.split.NumberedFileInputSplit;
+import org.datavec.image.transform.ImageTransform;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Random;
+
+@Slf4j
+public class UciSequenceDataFetcher extends CacheableExtractableDataSetFetcher {
+
+    public static int NUM_LABELS = 6;
+    public static int NUM_EXAMPLES = NUM_LABELS * 100;
+    private static String url = "https://archive.ics.uci.edu/ml/machine-learning-databases/synthetic_control-mld/synthetic_control.data";
+
+    public static void setURL(String url){
+        UciSequenceDataFetcher.url = url;
+    }
+
+    @Override
+    public String remoteDataUrl() {
+        return url;
+    }
+
+    @Override
+    public String remoteDataUrl(DataSetType type) {
+        return remoteDataUrl();
+    }
+
+    @Override
+    public String localCacheName() {
+        return "UCISequence_6";
+    }
+
+    @Override
+    public long expectedChecksum() {
+        return 104392751L;
+    }
+
+    @Override
+    public long expectedChecksum(DataSetType type) {
+        return expectedChecksum();
+    }
+
+    @Override
+    public CSVSequenceRecordReader getRecordReader(long rngSeed, int[] shape, DataSetType set, ImageTransform transform) {
+        return getRecordReader(rngSeed, set);
+    }
+
+    public CSVSequenceRecordReader getRecordReader(long rngSeed, DataSetType set) {
+
+        // check empty cache
+        File localCache = getLocalCacheDir();
+        deleteIfEmpty(localCache);
+
+        try {
+            if (!localCache.exists()) downloadAndExtract();
+        } catch (Exception e) {
+            throw new RuntimeException("Could not download UCI Sequence data", e);
+        }
+
+        File dataPath;
+
+        switch (set) {
+            case TRAIN:
+                dataPath = new File(localCache, "/train");
+                break;
+            case TEST:
+                dataPath = new File(localCache, "/test");
+                break;
+            case VALIDATION:
+                throw new IllegalArgumentException("You will need to manually iterate the directory, UCISequence data does not provide labels");
+
+            default:
+                dataPath = new File(localCache, "/train");
+        }
+
+        try {
+            downloadUCIData(dataPath);
+            CSVSequenceRecordReader data;
+            switch (set) {
+                case TRAIN:
+                    data = new CSVSequenceRecordReader(0, ", ");
+                    data.initialize(new NumberedFileInputSplit(dataPath.getAbsolutePath() + "/%d.csv", 0, 449));
+                    break;
+                case TEST:
+                    data = new CSVSequenceRecordReader(0, ", ");
+                    data.initialize(new NumberedFileInputSplit(dataPath.getAbsolutePath() + "/%d.csv", 450, 599));
+                    break;
+                default:
+                    data = new CSVSequenceRecordReader(0, ", ");
+                    data.initialize(new NumberedFileInputSplit(dataPath.getAbsolutePath() + "/%d.csv", 0, 449));
+            }
+
+            return data;
+        } catch (Exception e) {
+            throw new RuntimeException("Could not process UCI data", e);
+        }
+    }
+
+    private static void downloadUCIData(File dataPath) throws Exception {
+        //if (dataPath.exists()) return;
+
+        String data = IOUtils.toString(new URL(url), Charset.defaultCharset());
+        String[] lines = data.split("\n");
+
+        int lineCount = 0;
+        int index = 0;
+
+        ArrayList<String> linesList = new ArrayList<>();
+
+        for (String line : lines) {
+
+            // label value
+            int count = lineCount++ / 100;
+
+            // replace white space with commas and label value + new line
+            line = line.replaceAll("\\s+", ", " + count + "\n");
+
+            // add label to last number
+            line = line + ", " + count;
+            linesList.add(line);
+        }
+
+        // randomly shuffle data
+        Collections.shuffle(linesList, new Random(12345));
+
+        for (String line : linesList) {
+            File outPath = new File(dataPath, index + ".csv");
+            FileUtils.writeStringToFile(outPath, line, Charset.defaultCharset());
+            index++;
+        }
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/Cifar10DataSetIterator.java

@@ -0,0 +1,127 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.impl;
+
+import lombok.Getter;
+import org.apache.commons.io.FileUtils;
+import org.datavec.image.transform.ImageTransform;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator;
+import org.deeplearning4j.datasets.fetchers.Cifar10Fetcher;
+import org.deeplearning4j.datasets.fetchers.DataSetType;
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class Cifar10DataSetIterator extends RecordReaderDataSetIterator {
+
+    @Getter
+    protected DataSetPreProcessor preProcessor;
+
+    /**
+     * Create an iterator for the training set, with random iteration order (RNG seed fixed to 123)
+     *
+     * @param batchSize Minibatch size for the iterator
+     */
+    public Cifar10DataSetIterator(int batchSize) {
+        this(batchSize, null, DataSetType.TRAIN, null, 123);
+    }
+
+    /**
+     * * Create an iterator for the training or test set, with random iteration order (RNG seed fixed to 123)
+     *
+     * @param batchSize Minibatch size for the iterator
+     * @param set       The dataset (train or test)
+     */
+    public Cifar10DataSetIterator(int batchSize, DataSetType set) {
+        this(batchSize, null, set, null, 123);
+    }
+
+    /**
+     * Get the Tiny ImageNet iterator with specified train/test set, with random iteration order (RNG seed fixed to 123)
+     *
+     * @param batchSize Size of each patch
+     * @param imgDim Dimensions of desired output - for example, {64, 64}
+     * @param set Train, test, or validation
+     */
+    public Cifar10DataSetIterator(int batchSize, int[] imgDim, DataSetType set) {
+        this(batchSize, imgDim, set, null, 123);
+    }
+
+    /**
+     * Get the Tiny ImageNet iterator with specified train/test set and (optional) custom transform.
+     *
+     * @param batchSize Size of each patch
+     * @param imgDim Dimensions of desired output - for example, {64, 64}
+     * @param set Train, test, or validation
+     * @param imageTransform Additional image transform for output
+     * @param rngSeed random number generator seed to use when shuffling examples
+     */
+    public Cifar10DataSetIterator(int batchSize, int[] imgDim, DataSetType set,
+                                  ImageTransform imageTransform, long rngSeed) {
+        super(new Cifar10Fetcher().getRecordReader(rngSeed, imgDim, set, imageTransform), batchSize, 1, Cifar10Fetcher.NUM_LABELS);
+    }
+
+    /**
+     * Get the labels - either in "categories" (imagenet synsets format, "n01910747" or similar) or human-readable format,
+     * such as "jellyfish"
+     * @param categories If true: return category/synset format; false: return "human readable" label format
+     * @return Labels
+     */
+    public static List<String> getLabels(boolean categories){
+        List<String> rawLabels = new Cifar10DataSetIterator(1).getLabels();
+        if(categories){
+            return rawLabels;
+        }
+
+        //Otherwise, convert to human-readable format, using 'words.txt' file
+        File baseDir = DL4JResources.getDirectory(ResourceType.DATASET, Cifar10Fetcher.LOCAL_CACHE_NAME);
+        File labelFile = new File(baseDir, Cifar10Fetcher.LABELS_FILENAME);
+        List<String> lines;
+        try {
+            lines = FileUtils.readLines(labelFile, StandardCharsets.UTF_8);
+        } catch (IOException e){
+            throw new RuntimeException("Error reading label file", e);
+        }
+
+        Map<String,String> map = new HashMap<>();
+        for(String line : lines){
+            String[] split = line.split("\t");
+            map.put(split[0], split[1]);
+        }
+
+        List<String> outLabels = new ArrayList<>(rawLabels.size());
+        for(String s : rawLabels){
+            String s2 = map.get(s);
+            Preconditions.checkState(s2 != null, "Label \"%s\" not found in labels.txt file");
+            outLabels.add(s2);
+        }
+        return outLabels;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/EmnistDataSetIterator.java

@@ -0,0 +1,295 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.impl;
+
+import lombok.Getter;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.deeplearning4j.datasets.fetchers.EmnistDataFetcher;
+import org.eclipse.deeplearning4j.resources.utils.EMnistSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.BaseDatasetIterator;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class EmnistDataSetIterator extends BaseDatasetIterator {
+
+    private static final int NUM_COMPLETE_TRAIN = 697932;
+    private static final int NUM_COMPLETE_TEST = 116323;
+
+    private static final int NUM_MERGE_TRAIN = 697932;
+    private static final int NUM_MERGE_TEST = 116323;
+
+    private static final int NUM_BALANCED_TRAIN = 112800;
+    private static final int NUM_BALANCED_TEST = 18800;
+
+    private static final int NUM_DIGITS_TRAIN = 240000;
+    private static final int NUM_DIGITS_TEST = 40000;
+
+    private static final int NUM_LETTERS_TRAIN = 88800;
+    private static final int NUM_LETTERS_TEST = 14800;
+
+    private static final int NUM_MNIST_TRAIN = 60000;
+    private static final int NUM_MNIST_TEST = 10000;
+
+    private static final char[] LABELS_COMPLETE = new char[] {48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 65, 66, 67, 68,
+            69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99,
+            100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
+            120, 121, 122};
+
+    private static final char[] LABELS_MERGE = new char[] {48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 65, 66, 67, 68, 69,
+            70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 100,
+            101, 102, 103, 104, 110, 113, 114, 116};
+
+    private static final char[] LABELS_BALANCED = new char[] {48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 65, 66, 67, 68,
+            69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 100,
+            101, 102, 103, 104, 110, 113, 114, 116};
+
+    private static final char[] LABELS_DIGITS = new char[] {48, 49, 50, 51, 52, 53, 54, 55, 56, 57};
+
+    private static final char[] LABELS_LETTERS = new char[] {65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
+            80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90};
+
+    protected EMnistSet dataSet;
+    protected int batch, numExamples;
+    @Getter
+    protected DataSetPreProcessor preProcessor;
+
+    /**
+     * Create an EMNIST iterator with randomly shuffled data based on a random RNG seed
+     *
+     * @param dataSet Dataset (subset) to return
+     * @param batch   Batch size
+     * @param train   If true: use training set. If false: use test set
+     * @throws IOException If an error occurs when loading/downloading the dataset
+     */
+    public EmnistDataSetIterator(EMnistSet dataSet, int batch, boolean train) throws IOException {
+        this(dataSet, batch, train, System.currentTimeMillis());
+    }
+
+    /**
+     * Create an EMNIST iterator with randomly shuffled data based on a specified RNG seed
+     *
+     * @param dataSet   Dataset (subset) to return
+     * @param batchSize Batch size
+     * @param train     If true: use training set. If false: use test set
+     * @param seed      Random number generator seed
+     */
+    public EmnistDataSetIterator(EMnistSet dataSet, int batchSize, boolean train, long seed) throws IOException {
+        this(dataSet, batchSize, false, train, true, seed);
+    }
+
+    /**
+     * Get the specified number of MNIST examples (test or train set), with optional shuffling and binarization.
+     *
+     * @param batch    Size of each minibatch
+     * @param binarize whether to binarize the data or not (if false: normalize in range 0 to 1)
+     * @param train    Train vs. test set
+     * @param shuffle  whether to shuffle the examples
+     * @param rngSeed  random number generator seed to use when shuffling examples
+     */
+    public EmnistDataSetIterator(EMnistSet dataSet, int batch, boolean binarize, boolean train, boolean shuffle, long rngSeed, File topLevelDir)
+            throws IOException {
+        super(batch, numExamples(train, dataSet), new EmnistDataFetcher(dataSet, binarize, train, shuffle, rngSeed));
+        this.dataSet = dataSet;
+    }
+
+
+    /**
+     * Get the specified number of MNIST examples (test or train set), with optional shuffling and binarization.
+     *
+     * @param batch    Size of each minibatch
+     * @param binarize whether to binarize the data or not (if false: normalize in range 0 to 1)
+     * @param train    Train vs. test set
+     * @param shuffle  whether to shuffle the examples
+     * @param rngSeed  random number generator seed to use when shuffling examples
+     */
+    public EmnistDataSetIterator(EMnistSet dataSet, int batch, boolean binarize, boolean train, boolean shuffle, long rngSeed)
+            throws IOException {
+        this(dataSet,batch,binarize,train,shuffle,rngSeed, DL4JResources.getDirectory(ResourceType.DATASET,"emnist"));
+    }
+
+    private static int numExamples(boolean train, EMnistSet ds) {
+        if (train) {
+            return numExamplesTrain(ds);
+        } else {
+            return numExamplesTest(ds);
+        }
+    }
+
+    /**
+     * Get the number of training examples for the specified subset
+     *
+     * @param dataSet Subset to get
+     * @return Number of examples for the specified subset
+     */
+    public static int numExamplesTrain(EMnistSet dataSet) {
+        switch (dataSet) {
+            case COMPLETE:
+                return NUM_COMPLETE_TRAIN;
+            case MERGE:
+                return NUM_MERGE_TRAIN;
+            case BALANCED:
+                return NUM_BALANCED_TRAIN;
+            case LETTERS:
+                return NUM_LETTERS_TRAIN;
+            case DIGITS:
+                return NUM_DIGITS_TRAIN;
+            case MNIST:
+                return NUM_MNIST_TRAIN;
+            default:
+                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
+        }
+    }
+
+    /**
+     * Get the number of test examples for the specified subset
+     *
+     * @param dataSet Subset to get
+     * @return Number of examples for the specified subset
+     */
+    public static int numExamplesTest(EMnistSet dataSet) {
+        switch (dataSet) {
+            case COMPLETE:
+                return NUM_COMPLETE_TEST;
+            case MERGE:
+                return NUM_MERGE_TEST;
+            case BALANCED:
+                return NUM_BALANCED_TEST;
+            case LETTERS:
+                return NUM_LETTERS_TEST;
+            case DIGITS:
+                return NUM_DIGITS_TEST;
+            case MNIST:
+                return NUM_MNIST_TEST;
+            default:
+                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
+        }
+    }
+
+    /**
+     * Get the number of labels for the specified subset
+     *
+     * @param dataSet Subset to get
+     * @return Number of labels for the specified subset
+     */
+    public static int numLabels(EMnistSet dataSet) {
+        switch (dataSet) {
+            case COMPLETE:
+                return 62;
+            case MERGE:
+                return 47;
+            case BALANCED:
+                return 47;
+            case LETTERS:
+                return 26;
+            case DIGITS:
+                return 10;
+            case MNIST:
+                return 10;
+            default:
+                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
+        }
+    }
+
+    /**
+     * Get the labels as a character array
+     *
+     * @return Labels
+     */
+    public char[] getLabelsArrays() {
+        return getLabelsArray(dataSet);
+    }
+
+    /**
+     * Get the labels as a List<String>
+     *
+     * @return Labels
+     */
+    public List<String> getLabels() {
+        return getLabels(dataSet);
+    }
+
+    /**
+     * Get the label assignments for the given set as a character array.
+     *
+     * @param dataSet DataSet to get the label assignment for
+     * @return Label assignment and given dataset
+     */
+    public static char[] getLabelsArray(EMnistSet dataSet) {
+        switch (dataSet) {
+            case COMPLETE:
+                return LABELS_COMPLETE;
+            case MERGE:
+                return LABELS_MERGE;
+            case BALANCED:
+                return LABELS_BALANCED;
+            case LETTERS:
+                return LABELS_LETTERS;
+            case DIGITS:
+            case MNIST:
+                return LABELS_DIGITS;
+            default:
+                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
+        }
+    }
+
+    /**
+     * Get the label assignments for the given set as a List<String>
+     *
+     * @param dataSet DataSet to get the label assignment for
+     * @return Label assignment and given dataset
+     */
+    public static List<String> getLabels(EMnistSet dataSet) {
+        char[] c = getLabelsArray(dataSet);
+        List<String> l = new ArrayList<>(c.length);
+        for (char c2 : c) {
+            l.add(String.valueOf(c2));
+        }
+        return l;
+    }
+
+    /**
+     * Are the labels balanced in the training set (that is: are the number of examples for each label equal?)
+     *
+     * @param dataSet Set to get balanced value for
+     * @return True if balanced dataset, false otherwise
+     */
+    public static boolean isBalanced(EMnistSet dataSet) {
+        switch (dataSet) {
+            case COMPLETE:
+            case MERGE:
+            case LETTERS:
+                //Note: EMNIST docs claims letters is balanced, but this is not possible for training set:
+                // 88800 examples / 26 classes = 3418.46
+                return false;
+            case BALANCED:
+            case DIGITS:
+            case MNIST:
+                return true;
+            default:
+                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
+        }
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/IrisDataSetIterator.java

@@ -0,0 +1,63 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.impl;
+
+import org.deeplearning4j.datasets.fetchers.IrisDataFetcher;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.iterator.BaseDatasetIterator;
+
+public class IrisDataSetIterator extends BaseDatasetIterator {
+
+    /**
+     * 
+     */
+    private static final long serialVersionUID = -2022454995728680368L;
+
+    /**
+     * Create an iris iterator for full batch training - i.e., all 150 examples are included per minibatch
+     */
+    public IrisDataSetIterator(){
+        this(150, 150);
+    }
+
+    /**
+     * IrisDataSetIterator handles traversing through the Iris Data Set.
+     * @see <a href="https://archive.ics.uci.edu/ml/datasets/Iris">https://archive.ics.uci.edu/ml/datasets/Iris</a>
+     *
+     * @param batch Batch size
+     * @param numExamples Total number of examples
+     */
+    public IrisDataSetIterator(int batch, int numExamples) {
+        super(batch, numExamples, new IrisDataFetcher());
+    }
+
+
+    @Override
+    public DataSet next() {
+        fetcher.fetch(batch);
+        DataSet next = fetcher.next();
+        if(preProcessor != null) {
+            preProcessor.preProcess(next);
+        }
+
+        return next;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/LFWDataSetIterator.java

@@ -0,0 +1,106 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.impl;
+
+
+import org.datavec.api.io.labels.ParentPathLabelGenerator;
+import org.datavec.api.io.labels.PathLabelGenerator;
+import org.datavec.image.loader.LFWLoader;
+import org.datavec.image.transform.ImageTransform;
+import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator;
+
+import java.util.Random;
+
+public class LFWDataSetIterator extends RecordReaderDataSetIterator {
+
+    /** Loads subset of images with given imgDim returned by the generator. */
+    public LFWDataSetIterator(int[] imgDim) {
+        this(LFWLoader.SUB_NUM_IMAGES, LFWLoader.SUB_NUM_IMAGES, imgDim, LFWLoader.SUB_NUM_LABELS, false,
+                        new ParentPathLabelGenerator(), true, 1, null, new Random(System.currentTimeMillis()));
+    }
+
+    /** Loads images with given  batchSize, numExamples returned by the generator. */
+    public LFWDataSetIterator(int batchSize, int numExamples) {
+        this(batchSize, numExamples, new int[] {LFWLoader.HEIGHT, LFWLoader.WIDTH, LFWLoader.CHANNELS},
+                        LFWLoader.NUM_LABELS, false, LFWLoader.LABEL_PATTERN, true, 1, null,
+                        new Random(System.currentTimeMillis()));
+    }
+
+    /** Loads images with given  batchSize, numExamples, imgDim returned by the generator. */
+    public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim) {
+        this(batchSize, numExamples, imgDim, LFWLoader.NUM_LABELS, false, LFWLoader.LABEL_PATTERN, true, 1, null,
+                        new Random(System.currentTimeMillis()));
+    }
+
+    /** Loads images with given  batchSize, imgDim, useSubset, returned by the generator. */
+    public LFWDataSetIterator(int batchSize, int[] imgDim, boolean useSubset) {
+        this(batchSize, useSubset ? LFWLoader.SUB_NUM_IMAGES : LFWLoader.NUM_IMAGES, imgDim,
+                        useSubset ? LFWLoader.SUB_NUM_LABELS : LFWLoader.NUM_LABELS, useSubset, LFWLoader.LABEL_PATTERN,
+                        true, 1, null, new Random(System.currentTimeMillis()));
+    }
+
+    /** Loads images with given  batchSize, numExamples, imgDim, train, & splitTrainTest returned by the generator. */
+    public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, boolean train, double splitTrainTest) {
+        this(batchSize, numExamples, imgDim, LFWLoader.NUM_LABELS, false, LFWLoader.LABEL_PATTERN, train,
+                        splitTrainTest, null, new Random(System.currentTimeMillis()));
+    }
+
+    /** Loads images with given  batchSize, numExamples, numLabels, train, & splitTrainTest returned by the generator. */
+    public LFWDataSetIterator(int batchSize, int numExamples, int numLabels, boolean train, double splitTrainTest) {
+        this(batchSize, numExamples, new int[] {LFWLoader.HEIGHT, LFWLoader.WIDTH, LFWLoader.CHANNELS}, numLabels,
+                        false, null, train, splitTrainTest, null, new Random(System.currentTimeMillis()));
+    }
+
+    /** Loads images with given  batchSize, numExamples, imgDim, numLabels, useSubset, train, splitTrainTest & Random returned by the generator. */
+    public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset,
+                    boolean train, double splitTrainTest, Random rng) {
+        this(batchSize, numExamples, imgDim, numLabels, useSubset, LFWLoader.LABEL_PATTERN, train, splitTrainTest, null,
+                        rng);
+    }
+
+    /** Loads images with given  batchSize, numExamples, imgDim, numLabels, useSubset, train, splitTrainTest & Random returned by the generator. */
+    public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset,
+                    PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
+        this(batchSize, numExamples, imgDim, numLabels, useSubset, labelGenerator, train, splitTrainTest, null, rng);
+    }
+
+    /**
+     * Create LFW data specific iterator
+     * @param batchSize the batch size of the examples
+     * @param numExamples the overall number of examples
+     * @param imgDim an array of height, width and channels
+     * @param numLabels the overall number of examples
+     * @param useSubset use a subset of the LFWDataSet
+     * @param labelGenerator path label generator to use
+     * @param train true if use train value
+     * @param splitTrainTest the percentage to split data for train and remainder goes to test
+     * @param imageTransform how to transform the image
+    
+     * @param rng random number to lock in batch shuffling
+     * */
+    public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset,
+                    PathLabelGenerator labelGenerator, boolean train, double splitTrainTest,
+                    ImageTransform imageTransform, Random rng) {
+        super(new LFWLoader(imgDim, imageTransform, useSubset).getRecordReader(batchSize, numExamples, imgDim,
+                        numLabels, labelGenerator, train, splitTrainTest, rng), batchSize, 1, numLabels);
+    }
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/MnistDataSetIterator.java

@@ -0,0 +1,89 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.impl;
+
+import org.deeplearning4j.datasets.fetchers.MnistDataFetcher;
+import org.nd4j.linalg.dataset.api.iterator.BaseDatasetIterator;
+
+import java.io.File;
+import java.io.IOException;
+
+public class MnistDataSetIterator extends BaseDatasetIterator {
+
+    public MnistDataSetIterator(int batch, int numExamples) throws IOException {
+        this(batch, numExamples, false);
+    }
+
+    /**Get the specified number of examples for the MNIST training data set.
+     * @param batch the batch size of the examples
+     * @param numExamples the overall number of examples
+     * @param binarize whether to binarize mnist or not
+     * @throws IOException
+     */
+    public MnistDataSetIterator(int batch, int numExamples, boolean binarize) throws IOException {
+        this(batch, numExamples, binarize, true, false, 0);
+    }
+
+    /** Constructor to get the full MNIST data set (either test or train sets) without binarization (i.e., just normalization
+     * into range of 0 to 1), with shuffling based on a random seed.
+     * @param batchSize
+     * @param train
+     * @throws IOException
+     */
+    public MnistDataSetIterator(int batchSize, boolean train, int seed) throws IOException {
+        this(batchSize, (train ? MnistDataFetcher.NUM_EXAMPLES : MnistDataFetcher.NUM_EXAMPLES_TEST), false, train,
+                true, seed);
+    }
+
+    /**Get the specified number of MNIST examples (test or train set), with optional shuffling and binarization.
+     * @param batch Size of each patch
+     * @param numExamples total number of examples to load
+     * @param binarize whether to binarize the data or not (if false: normalize in range 0 to 1)
+     * @param train Train vs. test set
+     * @param shuffle whether to shuffle the examples
+     * @param rngSeed random number generator seed to use when shuffling examples
+     */
+    public MnistDataSetIterator(int batch, int numExamples, boolean binarize, boolean train, boolean shuffle,
+                                long rngSeed) throws IOException {
+        this(batch, numExamples, binarize,train,shuffle,rngSeed,null);
+    }
+
+
+    /**Get the specified number of MNIST examples (test or train set), with optional shuffling and binarization.
+     * @param batch Size of each patch
+     * @param numExamples total number of examples to load
+     * @param binarize whether to binarize the data or not (if false: normalize in range 0 to 1)
+     * @param train Train vs. test set
+     * @param shuffle whether to shuffle the examples
+     * @param rngSeed random number generator seed to use when shuffling examples
+     */
+    public MnistDataSetIterator(int batch, int numExamples, boolean binarize, boolean train, boolean shuffle,
+                                long rngSeed, File topLevelDir) throws IOException {
+        super(batch, numExamples, new MnistDataFetcher(binarize, train, shuffle, rngSeed, numExamples,topLevelDir));
+    }
+
+
+    public void close() {
+        MnistDataFetcher mnistDataFetcher = (MnistDataFetcher) fetcher;
+        mnistDataFetcher.close();
+    }
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/TinyImageNetDataSetIterator.java

@@ -0,0 +1,127 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.impl;
+
+import lombok.Getter;
+import org.apache.commons.io.FileUtils;
+import org.datavec.image.transform.ImageTransform;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator;
+import org.deeplearning4j.datasets.fetchers.DataSetType;
+import org.deeplearning4j.datasets.fetchers.TinyImageNetFetcher;
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class TinyImageNetDataSetIterator extends RecordReaderDataSetIterator {
+
+    @Getter
+    protected DataSetPreProcessor preProcessor;
+
+    /**
+     * Create an iterator for the training set, with random iteration order (RNG seed fixed to 123)
+     *
+     * @param batchSize Minibatch size for the iterator
+     */
+    public TinyImageNetDataSetIterator(int batchSize) {
+        this(batchSize, null, DataSetType.TRAIN, null, 123);
+    }
+
+    /**
+     * * Create an iterator for the training or test set, with random iteration order (RNG seed fixed to 123)
+     *
+     * @param batchSize Minibatch size for the iterator
+     * @param set       The dataset (train or test)
+     */
+    public TinyImageNetDataSetIterator(int batchSize, DataSetType set) {
+        this(batchSize, null, set, null, 123);
+    }
+
+    /**
+     * Get the Tiny ImageNet iterator with specified train/test set, with random iteration order (RNG seed fixed to 123)
+     *
+     * @param batchSize Size of each patch
+     * @param imgDim Dimensions of desired output - for example, {64, 64}
+     * @param set Train, test, or validation
+     */
+    public TinyImageNetDataSetIterator(int batchSize, int[] imgDim, DataSetType set) {
+        this(batchSize, imgDim, set, null, 123);
+    }
+
+    /**
+     * Get the Tiny ImageNet iterator with specified train/test set and (optional) custom transform.
+     *
+     * @param batchSize Size of each patch
+     * @param imgDim Dimensions of desired output - for example, {64, 64}
+     * @param set Train, test, or validation
+     * @param imageTransform Additional image transform for output
+     * @param rngSeed random number generator seed to use when shuffling examples
+     */
+    public TinyImageNetDataSetIterator(int batchSize, int[] imgDim, DataSetType set,
+                                       ImageTransform imageTransform, long rngSeed) {
+        super(new TinyImageNetFetcher().getRecordReader(rngSeed, imgDim, set, imageTransform), batchSize, 1, TinyImageNetFetcher.NUM_LABELS);
+    }
+
+    /**
+     * Get the labels - either in "categories" (imagenet synsets format, "n01910747" or similar) or human-readable format,
+     * such as "jellyfish"
+     * @param categories If true: return category/synset format; false: return "human readable" label format
+     * @return Labels
+     */
+    public static List<String> getLabels(boolean categories){
+        List<String> rawLabels = new TinyImageNetDataSetIterator(1).getLabels();
+        if(categories){
+            return rawLabels;
+        }
+
+        //Otherwise, convert to human-readable format, using 'words.txt' file
+        File baseDir = DL4JResources.getDirectory(ResourceType.DATASET, TinyImageNetFetcher.LOCAL_CACHE_NAME);
+        File labelFile = new File(baseDir, TinyImageNetFetcher.WORDS_FILENAME);
+        List<String> lines;
+        try {
+            lines = FileUtils.readLines(labelFile, StandardCharsets.UTF_8);
+        } catch (IOException e){
+            throw new RuntimeException("Error reading label file", e);
+        }
+
+        Map<String,String> map = new HashMap<>();
+        for(String line : lines){
+            String[] split = line.split("\t");
+            map.put(split[0], split[1]);
+        }
+
+        List<String> outLabels = new ArrayList<>(rawLabels.size());
+        for(String s : rawLabels){
+            String s2 = map.get(s);
+            Preconditions.checkState(s2 != null, "Label \"%s\" not found in labels.txt file");
+            outLabels.add(s2);
+        }
+        return outLabels;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/UciSequenceDataSetIterator.java

@@ -0,0 +1,62 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.impl;
+
+import org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator;
+import org.deeplearning4j.datasets.fetchers.DataSetType;
+import org.deeplearning4j.datasets.fetchers.UciSequenceDataFetcher;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+
+public class UciSequenceDataSetIterator extends SequenceRecordReaderDataSetIterator {
+
+    protected DataSetPreProcessor preProcessor;
+
+    /**
+     * Create an iterator for the training set, with the specified minibatch size. Randomized with RNG seed 123
+     *
+     * @param batchSize Minibatch size
+     */
+    public UciSequenceDataSetIterator(int batchSize) {
+        this(batchSize, DataSetType.TRAIN, 123);
+    }
+
+    /**
+     * Create an iterator for the training or test set, with the specified minibatch size. Randomized with RNG seed 123
+     *
+     * @param batchSize Minibatch size
+     * @param set       Set: training or test
+     */
+    public UciSequenceDataSetIterator(int batchSize, DataSetType set) {
+        this(batchSize, set, 123);
+    }
+
+    /**
+     * Create an iterator for the training or test set, with the specified minibatch size
+     *
+     * @param batchSize Minibatch size
+     * @param set       Set: training or test
+     * @param rngSeed   Random number generator seed to use for randomization
+     */
+    public UciSequenceDataSetIterator(int batchSize, DataSetType set, long rngSeed) {
+        super(new UciSequenceDataFetcher().getRecordReader(rngSeed, set), batchSize, UciSequenceDataFetcher.NUM_LABELS, 1);
+        // last parameter is index of label
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistDbFile.java

@@ -0,0 +1,125 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.mnist;
+
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+public abstract class MnistDbFile extends RandomAccessFile {
+    private int count;
+
+
+    /**
+     * Creates new instance and reads the header information.
+     * 
+     * @param name
+     *            the system-dependent filename
+     * @param mode
+     *            the access mode
+     * @throws IOException
+     * @throws FileNotFoundException
+     * @see RandomAccessFile
+     */
+    public MnistDbFile(String name, String mode) throws IOException {
+        super(name, mode);
+        if (getMagicNumber() != readInt()) {
+            throw new RuntimeException(
+                            "This MNIST DB file " + name + " should start with the number " + getMagicNumber() + ".");
+        }
+        count = readInt();
+    }
+
+    /**
+     * MNIST DB files start with unique integer number.
+     * 
+     * @return integer number that should be found in the beginning of the file.
+     */
+    protected abstract int getMagicNumber();
+
+    /**
+     * The current entry index.
+     * 
+     * @return long
+     * @throws IOException
+     */
+    public long getCurrentIndex() throws IOException {
+        return (getFilePointer() - getHeaderSize()) / getEntryLength() + 1;
+    }
+
+    /**
+     * Set the required current entry index.
+     * 
+     * @param curr
+     *            the entry index
+     */
+    public void setCurrentIndex(long curr) {
+        try {
+            if (curr < 0 || curr > count) {
+                throw new RuntimeException(curr + " is not in the range 0 to " + count);
+            }
+            seek(getHeaderSize() + curr * getEntryLength());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public int getHeaderSize() {
+        return 8; // two integers
+    }
+
+    /**
+     * Number of bytes for each entry.
+     * Defaults to 1.
+     * 
+     * @return int
+     */
+    public int getEntryLength() {
+        return 1;
+    }
+
+    /**
+     * Move to the next entry.
+     * 
+     * @throws IOException
+     */
+    public void next() throws IOException {
+        if (getCurrentIndex() < count) {
+            skipBytes(getEntryLength());
+        }
+    }
+
+    /**
+     * Move to the previous entry.
+     * 
+     * @throws IOException
+     */
+    public void prev() throws IOException {
+        if (getCurrentIndex() > 0) {
+            seek(getFilePointer() - getEntryLength());
+        }
+    }
+
+    public int getCount() {
+        return count;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistImageFile.java

@@ -0,0 +1,130 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.mnist;
+
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+
+public class MnistImageFile extends MnistDbFile {
+    private int rows;
+    private int cols;
+
+    /**
+     * Creates new MNIST database image file ready for reading.
+     * 
+     * @param name
+     *            the system-dependent filename
+     * @param mode
+     *            the access mode
+     * @throws IOException
+     * @throws FileNotFoundException
+     */
+    public MnistImageFile(String name, String mode) throws IOException {
+        super(name, mode);
+
+        // read header information
+        rows = readInt();
+        cols = readInt();
+    }
+
+    /**
+     * Reads the image at the current position.
+     * 
+     * @return matrix representing the image
+     * @throws IOException
+     */
+    public int[][] readImage() throws IOException {
+        int[][] dat = new int[getRows()][getCols()];
+        for (int i = 0; i < getCols(); i++) {
+            for (int j = 0; j < getRows(); j++) {
+                dat[i][j] = readUnsignedByte();
+            }
+        }
+        return dat;
+    }
+
+    /** Read the specified number of images from the current position, to a byte[nImages][rows*cols]
+     * Note that MNIST data set is stored as unsigned bytes; this method returns signed bytes without conversion
+     * (i.e., same bits, but requires conversion before use)
+     * @param nImages Number of images
+     */
+    public byte[][] readImagesUnsafe(int nImages) throws IOException {
+        byte[][] out = new byte[nImages][0];
+        for (int i = 0; i < nImages; i++) {
+            out[i] = new byte[rows * cols];
+            read(out[i]);
+        }
+        return out;
+    }
+
+    /**
+     * Move the cursor to the next image.
+     * 
+     * @throws IOException
+     */
+    public void nextImage() throws IOException {
+        super.next();
+    }
+
+    /**
+     * Move the cursor to the previous image.
+     * 
+     * @throws IOException
+     */
+    public void prevImage() throws IOException {
+        super.prev();
+    }
+
+    @Override
+    protected int getMagicNumber() {
+        return 2051;
+    }
+
+    /**
+     * Number of rows per image.
+     * 
+     * @return int
+     */
+    public int getRows() {
+        return rows;
+    }
+
+    /**
+     * Number of columns per image.
+     * 
+     * @return int
+     */
+    public int getCols() {
+        return cols;
+    }
+
+    @Override
+    public int getEntryLength() {
+        return cols * rows;
+    }
+
+    @Override
+    public int getHeaderSize() {
+        return super.getHeaderSize() + 8; // to more integers - rows and columns
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistLabelFile.java

@@ -0,0 +1,71 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.mnist;
+
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+
+/**
+ * 
+ * MNIST database label file.
+ * 
+ */
+public class MnistLabelFile extends MnistDbFile {
+
+    /**
+     * Creates new MNIST database label file ready for reading.
+     * 
+     * @param name
+     *            the system-dependent filename
+     * @param mode
+     *            the access mode
+     * @throws IOException
+     * @throws FileNotFoundException
+     */
+    public MnistLabelFile(String name, String mode) throws IOException {
+        super(name, mode);
+    }
+
+    /**
+     * Reads the integer at the current position.
+     * 
+     * @return integer representing the label
+     * @throws IOException
+     */
+    public int readLabel() throws IOException {
+        return readUnsignedByte();
+    }
+
+    /** Read the specified number of labels from the current position*/
+    public int[] readLabels(int num) throws IOException {
+        int[] out = new int[num];
+        for (int i = 0; i < num; i++)
+            out[i] = readLabel();
+        return out;
+    }
+
+    @Override
+    protected int getMagicNumber() {
+        return 2049;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistManager.java

@@ -0,0 +1,188 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.mnist;
+
+
+import lombok.SneakyThrows;
+import org.deeplearning4j.datasets.fetchers.MnistDataFetcher;
+import org.nd4j.common.base.Preconditions;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+
+
+public class MnistManager {
+    MnistImageFile images;
+    private MnistLabelFile labels;
+
+    private byte[][] imagesArr;
+    private int[] labelsArr;
+    private static final int HEADER_SIZE = 8;
+
+    /**
+     * Writes the given image in the given file using the PPM data format.
+     *
+     * @param image
+     * @param ppmFileName
+     * @throws IOException
+     */
+    public static void writeImageToPpm(int[][] image, String ppmFileName) throws IOException {
+        try (BufferedWriter ppmOut = new BufferedWriter(new FileWriter(ppmFileName))) {
+            int rows = image.length;
+            int cols = image[0].length;
+            ppmOut.write("P3\n");
+            ppmOut.write("" + rows + " " + cols + " 255\n");
+            for (int i = 0; i < rows; i++) {
+                StringBuilder s = new StringBuilder();
+                for (int j = 0; j < cols; j++) {
+                    s.append(image[i][j] + " " + image[i][j] + " " + image[i][j] + "  ");
+                }
+                ppmOut.write(s.toString());
+            }
+        }
+
+    }
+
+    @SneakyThrows
+    public long getCurrent() {
+        return labels.getCurrentIndex();
+    }
+
+
+
+    /**
+     * Constructs an instance managing the two given data files. Supports
+     * <code>NULL</code> value for one of the arguments in case reading only one
+     * of the files (images and labels) is required.
+     *
+     * @param imagesFile
+     *            Can be <code>NULL</code>. In that case all future operations
+     *            using that file will fail.
+     * @param labelsFile
+     *            Can be <code>NULL</code>. In that case all future operations
+     *            using that file will fail.
+     * @throws IOException
+     */
+    public MnistManager(String imagesFile, String labelsFile, boolean train) throws IOException {
+        this(imagesFile, labelsFile, train ? MnistDataFetcher.NUM_EXAMPLES : MnistDataFetcher.NUM_EXAMPLES_TEST);
+    }
+
+
+
+    public MnistManager(String imagesFile, String labelsFile, int numExamples) throws IOException {
+        if (imagesFile != null) {
+            images = new MnistImageFile(imagesFile, "r");
+            imagesArr = images.readImagesUnsafe(numExamples);
+        }
+        if (labelsFile != null) {
+            labels = new MnistLabelFile(labelsFile, "r");
+            labelsArr = labels.readLabels(numExamples);
+        }
+    }
+
+    public MnistManager(String imagesFile, String labelsFile) throws IOException {
+        this(imagesFile, labelsFile, true);
+    }
+
+    /**
+     * Reads the current image.
+     *
+     * @return matrix
+     * @throws IOException
+     */
+    public int[][] readImage() throws IOException {
+        if (images == null) {
+            throw new IllegalStateException("Images file not initialized.");
+        }
+        return images.readImage();
+    }
+
+    public byte[] readImageUnsafe(int i) {
+        Preconditions.checkArgument(i < imagesArr.length);
+        return imagesArr[i];
+    }
+
+    /**
+     * Set the position to be read.
+     *
+     * @param index
+     */
+    public void setCurrent(int index) {
+        images.setCurrentIndex(index);
+        labels.setCurrentIndex(index);
+    }
+
+    /**
+     * Reads the current label.
+     *
+     * @return int
+     * @throws IOException
+     */
+    public int readLabel() throws IOException {
+        if (labels == null) {
+            throw new IllegalStateException("labels file not initialized.");
+        }
+        return labels.readLabel();
+    }
+
+    public int readLabel(int i) {
+        return labelsArr[i];
+    }
+
+    /**
+     * Get the underlying images file as {@link MnistImageFile}.
+     *
+     * @return {@link MnistImageFile}.
+     */
+    public MnistImageFile getImages() {
+        return images;
+    }
+
+    /**
+     * Get the underlying labels file as {@link MnistLabelFile}.
+     *
+     * @return {@link MnistLabelFile}.
+     */
+    public MnistLabelFile getLabels() {
+        return labels;
+    }
+
+    /**
+     * Close any resources opened by the manager.
+     */
+    public void close() {
+        if (images != null) {
+            try {
+                images.close();
+            } catch (IOException e) {
+            }
+            images = null;
+        }
+        if (labels != null) {
+            try {
+                labels.close();
+            } catch (IOException e) {
+            }
+            labels = null;
+        }
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java9/module-info.java

@@ -0,0 +1,15 @@
+open module deeplearning4j.datasets {
+    requires commons.io;
+    requires lombok;
+    requires nd4j.common;
+    requires slf4j.api;
+    requires datavec.api;
+    requires datavec.data.image;
+    requires deeplearning4j.datavec.iterators;
+    requires nd4j.api;
+    requires resources;
+    exports org.deeplearning4j.datasets.base;
+    exports org.deeplearning4j.datasets.fetchers;
+    exports org.deeplearning4j.datasets.iterator.impl;
+    exports org.deeplearning4j.datasets.mnist;
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/pom.xml

@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ /* ******************************************************************************
+  ~  *
+  ~  *
+  ~  * This program and the accompanying materials are made available under the
+  ~  * terms of the Apache License, Version 2.0 which is available at
+  ~  * https://www.apache.org/licenses/LICENSE-2.0.
+  ~  *
+  ~  *  See the NOTICE file distributed with this work for additional
+  ~  *  information regarding copyright ownership.
+  ~  * Unless required by applicable law or agreed to in writing, software
+  ~  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  ~  * License for the specific language governing permissions and limitations
+  ~  * under the License.
+  ~  *
+  ~  * SPDX-License-Identifier: Apache-2.0
+  ~  ******************************************************************************/
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.eclipse.deeplearning4j</groupId>
+        <artifactId>deeplearning4j-data</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>deeplearning4j-datavec-iterators</artifactId>
+    <packaging>jar</packaging>
+
+    <name>deeplearning4j-datavec-iterators</name>
+
+    <properties>
+        <module.name>deeplearning4j.datavec.iterators</module.name>
+    </properties>
+
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.moditect</groupId>
+                <artifactId>moditect-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>datavec-api</artifactId>
+            <version>${datavec.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>nd4j-api</artifactId>
+        </dependency>
+    </dependencies>
+
+
+</project>

deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/RecordReaderDataSetIterator.java

@@ -0,0 +1,580 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.datavec;
+
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.datavec.api.io.WritableConverter;
+import org.datavec.api.io.converters.SelfWritableConverter;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataComposableMap;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.records.reader.SequenceRecordReader;
+import org.datavec.api.records.reader.impl.ConcatenatingRecordReader;
+import org.datavec.api.records.reader.impl.collection.CollectionRecordReader;
+import org.datavec.api.writable.Writable;
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.factory.Nd4j;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+
+@Slf4j
+public class RecordReaderDataSetIterator implements DataSetIterator {
+    private static final String READER_KEY = "reader";
+    @Getter
+    protected RecordReader recordReader;
+    protected WritableConverter converter;
+    protected int batchSize = 10;
+    protected int maxNumBatches = -1;
+    protected int batchNum = 0;
+    protected int labelIndex = -1;
+    protected int labelIndexTo = -1;
+    protected int numPossibleLabels = -1;
+    protected Iterator<List<Writable>> sequenceIter;
+    protected DataSet last;
+    protected boolean useCurrent = false;
+    protected boolean regression = false;
+    @Getter
+    protected DataSetPreProcessor preProcessor;
+
+    @Getter
+    private boolean collectMetaData = false;
+
+    private RecordReaderMultiDataSetIterator underlying;
+    private boolean underlyingIsDisjoint;
+
+    /**
+     * Constructor for classification, where:<br>
+     * (a) the label index is assumed to be the very last Writable/column, and<br>
+     * (b) the number of classes is inferred from RecordReader.getLabels()<br>
+     * Note that if RecordReader.getLabels() returns null, no output labels will be produced
+     *
+     * @param recordReader Record reader to use as the source of data
+     * @param batchSize    Minibatch size, for each call of .next()
+     */
+    public RecordReaderDataSetIterator(RecordReader recordReader, int batchSize) {
+        this(recordReader, new SelfWritableConverter(), batchSize, -1, -1,
+                recordReader.getLabels() == null ? -1 : recordReader.getLabels().size(), -1, false);
+    }
+
+    /**
+     * Main constructor for classification. This will convert the input class index (at position labelIndex, with integer
+     * values 0 to numPossibleLabels-1 inclusive) to the appropriate one-hot output/labels representation.
+     *
+     * @param recordReader         RecordReader: provides the source of the data
+     * @param batchSize            Batch size (number of examples) for the output DataSet objects
+     * @param labelIndex           Index of the label Writable (usually an IntWritable), as obtained by recordReader.next()
+     * @param numPossibleLabels    Number of classes (possible labels) for classification
+     */
+    public RecordReaderDataSetIterator(RecordReader recordReader, int batchSize, int labelIndex,
+                                       int numPossibleLabels) {
+        this(recordReader, new SelfWritableConverter(), batchSize, labelIndex, labelIndex, numPossibleLabels, -1, false);
+    }
+
+    /**
+     * Constructor for classification, where the maximum number of returned batches is limited to the specified value
+     *
+     * @param recordReader      the recordreader to use
+     * @param labelIndex        the index/column of the label (for classification)
+     * @param numPossibleLabels the number of possible labels for classification. Not used if regression == true
+     * @param maxNumBatches     The maximum number of batches to return between resets. Set to -1 to return all available data
+     */
+    public RecordReaderDataSetIterator(RecordReader recordReader, int batchSize, int labelIndex, int numPossibleLabels,
+                                       int maxNumBatches) {
+        this(recordReader, new SelfWritableConverter(), batchSize, labelIndex, labelIndex, numPossibleLabels, maxNumBatches, false);
+    }
+
+    /**
+     * Main constructor for multi-label regression (i.e., regression with multiple outputs). Can also be used for single
+     * output regression with labelIndexFrom == labelIndexTo
+     *
+     * @param recordReader      RecordReader to get data from
+     * @param labelIndexFrom    Index of the first regression target
+     * @param labelIndexTo      Index of the last regression target, inclusive
+     * @param batchSize         Minibatch size
+     * @param regression        Require regression = true. Mainly included to avoid clashing with other constructors previously defined :/
+     */
+    public RecordReaderDataSetIterator(RecordReader recordReader, int batchSize, int labelIndexFrom, int labelIndexTo,
+                                       boolean regression) {
+		this(recordReader, new SelfWritableConverter(), batchSize, labelIndexFrom, labelIndexTo, -1, -1, regression);
+		if (!regression) {
+            throw new IllegalArgumentException("This constructor is only for creating regression iterators. " +
+                                               "If you're doing classification you need to use another constructor that " +
+                                               "(implicitly) specifies numPossibleLabels");
+        }
+    }
+
+
+    /**
+     * Main constructor
+     *
+     * @param recordReader      the recordreader to use
+     * @param converter         Converter. May be null.
+     * @param batchSize         Minibatch size - number of examples returned for each call of .next()
+     * @param labelIndexFrom    the index of the label (for classification), or the first index of the labels for multi-output regression
+     * @param labelIndexTo      only used if regression == true. The last index <i>inclusive</i> of the multi-output regression
+     * @param numPossibleLabels the number of possible labels for classification. Not used if regression == true
+     * @param maxNumBatches     Maximum number of batches to return
+     * @param regression        if true: regression. If false: classification (assume labelIndexFrom is the class it belongs to)
+     */
+    public RecordReaderDataSetIterator(RecordReader recordReader, WritableConverter converter, int batchSize,
+                                       int labelIndexFrom, int labelIndexTo, int numPossibleLabels, int maxNumBatches,
+                                       boolean regression) {
+        this.recordReader = recordReader;
+        this.converter = converter;
+        this.batchSize = batchSize;
+        this.maxNumBatches = maxNumBatches;
+        this.labelIndex = labelIndexFrom;
+        this.labelIndexTo = labelIndexTo;
+        this.numPossibleLabels = numPossibleLabels;
+        this.regression = regression;
+    }
+
+
+    protected RecordReaderDataSetIterator(Builder b){
+        this.recordReader = b.recordReader;
+        this.converter = b.converter;
+        this.batchSize = b.batchSize;
+        this.maxNumBatches = b.maxNumBatches;
+        this.labelIndex = b.labelIndex;
+        this.labelIndexTo = b.labelIndexTo;
+        this.numPossibleLabels = b.numPossibleLabels;
+        this.regression = b.regression;
+        this.preProcessor = b.preProcessor;
+        this.collectMetaData = b.collectMetaData;
+    }
+
+    /**
+     * When set to true: metadata for  the current examples will be present in the returned DataSet.
+     * Disabled by default.
+     *
+     * @param collectMetaData Whether to collect metadata or  not
+     */
+    public void setCollectMetaData(boolean collectMetaData) {
+        if (underlying != null) {
+            underlying.setCollectMetaData(collectMetaData);
+        }
+        this.collectMetaData = collectMetaData;
+    }
+
+    private void initializeUnderlying() {
+        if (underlying == null) {
+            Record next = recordReader.nextRecord();
+            initializeUnderlying(next);
+        }
+    }
+
+    private void initializeUnderlying(Record next) {
+        int totalSize = next.getRecord().size();
+
+        //allow people to specify label index as -1 and infer the last possible label
+        if (numPossibleLabels >= 1 && labelIndex < 0) {
+            labelIndex = totalSize - 1;
+            labelIndexTo = labelIndex;
+        }
+
+        if(recordReader.resetSupported()) {
+            recordReader.reset();
+        } else {
+            //Hack around the fact that we need the first record to initialize the underlying RRMDSI, but can't reset
+            // the original reader
+            recordReader = new ConcatenatingRecordReader(
+                    new CollectionRecordReader(Collections.singletonList(next.getRecord())),
+                    recordReader);
+        }
+
+        RecordReaderMultiDataSetIterator.Builder builder = new RecordReaderMultiDataSetIterator.Builder(batchSize);
+        if (recordReader instanceof SequenceRecordReader) {
+            builder.addSequenceReader(READER_KEY, (SequenceRecordReader) recordReader);
+        } else {
+            builder.addReader(READER_KEY, recordReader);
+        }
+
+        if (regression) {
+            builder.addOutput(READER_KEY, labelIndex, labelIndexTo);
+        } else if (numPossibleLabels >= 1) {
+            builder.addOutputOneHot(READER_KEY, labelIndex, numPossibleLabels);
+        }
+
+        //Inputs: assume to be all the other writables
+        //In general: can't assume label indices are all at the start or end (event though 99% of the time they are)
+        //If they are: easy. If not: use 2 inputs in the underlying as a workaround, and concat them
+
+        if (labelIndex >= 0 && (labelIndex == 0 || labelIndexTo == totalSize - 1)) {
+            //Labels are first or last -> one input in underlying
+            int inputFrom;
+            int inputTo;
+            if (labelIndex < 0) {
+                //No label
+                inputFrom = 0;
+                inputTo = totalSize - 1;
+            } else if (labelIndex == 0) {
+                inputFrom = labelIndexTo + 1;
+                inputTo = totalSize - 1;
+            } else {
+                inputFrom = 0;
+                inputTo = labelIndex - 1;
+            }
+
+            builder.addInput(READER_KEY, inputFrom, inputTo);
+
+            underlyingIsDisjoint = false;
+        } else if (labelIndex >= 0) {
+            Preconditions.checkState(labelIndex < next.getRecord().size(),
+                    "Invalid label (from) index: index must be in range 0 to first record size of (0 to %s inclusive), got %s", next.getRecord().size()-1, labelIndex);
+            Preconditions.checkState(labelIndexTo < next.getRecord().size(),
+                    "Invalid label (to) index: index must be in range 0 to first record size of (0 to %s inclusive), got %s", next.getRecord().size()-1, labelIndexTo);
+
+
+            //Multiple inputs
+            int firstFrom = 0;
+            int firstTo = labelIndex - 1;
+            int secondFrom = labelIndexTo + 1;
+            int secondTo = totalSize - 1;
+
+            builder.addInput(READER_KEY, firstFrom, firstTo);
+            builder.addInput(READER_KEY, secondFrom, secondTo);
+
+            underlyingIsDisjoint = true;
+        } else {
+            //No labels - only features
+            builder.addInput(READER_KEY);
+            underlyingIsDisjoint = false;
+        }
+
+
+        underlying = builder.build();
+
+        if (collectMetaData) {
+            underlying.setCollectMetaData(true);
+        }
+    }
+
+    private DataSet mdsToDataSet(MultiDataSet mds) {
+        INDArray f;
+        INDArray fm;
+        if (underlyingIsDisjoint) {
+            //Rare case: 2 input arrays -> concat
+            INDArray f1 = getOrNull(mds.getFeatures(), 0);
+            INDArray f2 = getOrNull(mds.getFeatures(), 1);
+            fm = getOrNull(mds.getFeaturesMaskArrays(), 0); //Per-example masking only on the input -> same for both
+
+            //Can assume 2d features here
+            f = Nd4j.hstack(f1, f2);
+        } else {
+            //Standard case
+            f = getOrNull(mds.getFeatures(), 0);
+            fm = getOrNull(mds.getFeaturesMaskArrays(), 0);
+        }
+
+        INDArray l = getOrNull(mds.getLabels(), 0);
+        INDArray lm = getOrNull(mds.getLabelsMaskArrays(), 0);
+
+        DataSet ds = new DataSet(f, l, fm, lm);
+
+        if (collectMetaData) {
+            List<Serializable> temp = mds.getExampleMetaData();
+            List<Serializable> temp2 = new ArrayList<>(temp.size());
+            for (Serializable s : temp) {
+                RecordMetaDataComposableMap m = (RecordMetaDataComposableMap) s;
+                temp2.add(m.getMeta().get(READER_KEY));
+            }
+            ds.setExampleMetaData(temp2);
+        }
+
+        //Edge case, for backward compatibility:
+        //If labelIdx == -1 && numPossibleLabels == -1 -> no labels -> set labels array to features array
+        if (labelIndex == -1 && numPossibleLabels == -1 && ds.getLabels() == null) {
+            ds.setLabels(ds.getFeatures());
+        }
+
+        if (preProcessor != null) {
+            preProcessor.preProcess(ds);
+        }
+
+        return ds;
+    }
+
+
+    @Override
+    public DataSet next(int num) {
+        if (useCurrent) {
+            useCurrent = false;
+            if (preProcessor != null)
+                preProcessor.preProcess(last);
+            return last;
+        }
+
+        if (underlying == null) {
+            initializeUnderlying();
+        }
+
+
+        batchNum++;
+        return mdsToDataSet(underlying.next(num));
+    }
+
+    //Package private
+    static INDArray getOrNull(INDArray[] arr, int idx) {
+        if (arr == null || arr.length == 0) {
+            return null;
+        }
+        return arr[idx];
+    }
+
+    @Override
+    public int inputColumns() {
+        if (last == null) {
+            DataSet next = next();
+            last = next;
+            useCurrent = true;
+            return next.numInputs();
+        } else
+            return last.numInputs();
+    }
+
+    @Override
+    public int totalOutcomes() {
+        if (last == null) {
+            DataSet next = next();
+            last = next;
+            useCurrent = true;
+            return next.numOutcomes();
+        } else
+            return last.numOutcomes();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        if(underlying == null){
+            initializeUnderlying();
+        }
+        return underlying.resetSupported();
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return true;
+    }
+
+    @Override
+    public void reset() {
+        batchNum = 0;
+        if (underlying != null) {
+            underlying.reset();
+        }
+
+        last = null;
+        useCurrent = false;
+    }
+
+    @Override
+    public int batch() {
+        return batchSize;
+    }
+
+    @Override
+    public void setPreProcessor(org.nd4j.linalg.dataset.api.DataSetPreProcessor preProcessor) {
+        this.preProcessor = preProcessor;
+    }
+
+    @Override
+    public boolean hasNext() {
+        return (((sequenceIter != null && sequenceIter.hasNext()) || recordReader.hasNext())
+                && (maxNumBatches < 0 || batchNum < maxNumBatches));
+    }
+
+    @Override
+    public DataSet next() {
+        return next(batchSize);
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return recordReader.getLabels();
+    }
+
+    /**
+     * Load a single example to a DataSet, using the provided RecordMetaData.
+     * Note that it is more efficient to load multiple instances at once, using {@link #loadFromMetaData(List)}
+     *
+     * @param recordMetaData RecordMetaData to load from. Should have been produced by the given record reader
+     * @return DataSet with the specified example
+     * @throws IOException If an error occurs during loading of the data
+     */
+    public DataSet loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return loadFromMetaData(Collections.singletonList(recordMetaData));
+    }
+
+    /**
+     * Load a multiple examples to a DataSet, using the provided RecordMetaData instances.
+     *
+     * @param list List of RecordMetaData instances to load from. Should have been produced by the record reader provided
+     *             to the RecordReaderDataSetIterator constructor
+     * @return DataSet with the specified examples
+     * @throws IOException If an error occurs during loading of the data
+     */
+    public DataSet loadFromMetaData(List<RecordMetaData> list) throws IOException {
+        if (underlying == null) {
+            Record r = recordReader.loadFromMetaData(list.get(0));
+            initializeUnderlying(r);
+        }
+
+        //Convert back to composable:
+        List<RecordMetaData> l = new ArrayList<>(list.size());
+        for (RecordMetaData m : list) {
+            l.add(new RecordMetaDataComposableMap(Collections.singletonMap(READER_KEY, m)));
+        }
+        MultiDataSet m = underlying.loadFromMetaData(l);
+
+        return mdsToDataSet(m);
+    }
+
+    /**
+     * Builder class for RecordReaderDataSetIterator
+     */
+    public static class Builder {
+
+        protected RecordReader recordReader;
+        protected WritableConverter converter;
+        protected int batchSize;
+        protected int maxNumBatches = -1;
+        protected int labelIndex = -1;
+        protected int labelIndexTo = -1;
+        protected int numPossibleLabels = -1;
+        protected boolean regression = false;
+        protected DataSetPreProcessor preProcessor;
+        private boolean collectMetaData = false;
+
+        private boolean clOrRegCalled = false;
+
+        /**
+         *
+         * @param rr        Underlying record reader to source data from
+         * @param batchSize Batch size to use
+         */
+        public Builder(@NonNull RecordReader rr, int batchSize){
+            this.recordReader = rr;
+            this.batchSize = batchSize;
+        }
+
+        public Builder writableConverter(WritableConverter converter){
+            this.converter = converter;
+            return this;
+        }
+
+        /**
+         * Optional argument, usually not used. If set, can be used to limit the maximum number of minibatches that
+         * will be returned (between resets). If not set, will always return as many minibatches as there is data
+         * available.
+         *
+         * @param maxNumBatches Maximum number of minibatches per epoch / reset
+         */
+        public Builder maxNumBatches(int maxNumBatches){
+            this.maxNumBatches = maxNumBatches;
+            return this;
+        }
+
+        /**
+         * Use this for single output regression (i.e., 1 output/regression target)
+         *
+         * @param labelIndex Column index that contains the regression target (indexes start at 0)
+         */
+        public Builder regression(int labelIndex){
+            return regression(labelIndex, labelIndex);
+        }
+
+        /**
+         * Use this for multiple output regression (1 or more output/regression targets). Note that all regression
+         * targets must be contiguous (i.e., positions x to y, without gaps)
+         *
+         * @param labelIndexFrom Column index of the first regression target (indexes start at 0)
+         * @param labelIndexTo   Column index of the last regression target (inclusive)
+         */
+        public Builder regression(int labelIndexFrom, int labelIndexTo){
+            this.labelIndex = labelIndexFrom;
+            this.labelIndexTo = labelIndexTo;
+            this.regression = true;
+            clOrRegCalled = true;
+            return this;
+        }
+
+        /**
+         * Use this for classification
+         *
+         * @param labelIndex Index that contains the label index. Column (indexes start from 0) be an integer value,
+         *                   and contain values 0 to numClasses-1
+         * @param numClasses Number of label classes (i.e., number of categories/classes in the dataset)
+         */
+        public Builder classification(int labelIndex, int numClasses){
+            this.labelIndex = labelIndex;
+            this.labelIndexTo = labelIndex;
+            this.numPossibleLabels = numClasses;
+            this.regression = false;
+            clOrRegCalled = true;
+            return this;
+        }
+
+        /**
+         * Optional arg. Allows the preprocessor to be set
+         * @param preProcessor Preprocessor to use
+         */
+        public Builder preProcessor(DataSetPreProcessor preProcessor){
+            this.preProcessor = preProcessor;
+            return this;
+        }
+
+        /**
+         * When set to true: metadata for  the current examples will be present in the returned DataSet.
+         * Disabled by default.
+         *
+         * @param collectMetaData Whether metadata should be collected or not
+         */
+        public Builder collectMetaData(boolean collectMetaData){
+            this.collectMetaData = collectMetaData;
+            return this;
+        }
+
+        public RecordReaderDataSetIterator build(){
+            return new RecordReaderDataSetIterator(this);
+        }
+
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/SequenceRecordReaderDataSetIterator.java

@@ -0,0 +1,480 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.datavec;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.datavec.api.records.SequenceRecord;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataComposable;
+import org.datavec.api.records.metadata.RecordMetaDataComposableMap;
+import org.datavec.api.records.reader.SequenceRecordReader;
+import org.deeplearning4j.datasets.datavec.exception.ZeroLengthSequenceException;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.indexing.INDArrayIndex;
+import org.nd4j.linalg.indexing.NDArrayIndex;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.*;
+
+public class SequenceRecordReaderDataSetIterator implements DataSetIterator {
+    /**Alignment mode for dealing with input/labels of differing lengths (for example, one-to-many and many-to-one type situations).
+     * For example, might have 10 time steps total but only one label at end for sequence classification.<br>
+     * Currently supported modes:<br>
+     * <b>EQUAL_LENGTH</b>: Default. Assume that label and input time series are of equal length, and all examples are of
+     * the same length<br>
+     * <b>ALIGN_START</b>: Align the label/input time series at the first time step, and zero pad either the labels or
+     * the input at the end<br>
+     * <b>ALIGN_END</b>: Align the label/input at the last time step, zero padding either the input or the labels as required<br>
+     *
+     * Note 1: When the time series for each example are of different lengths, the shorter time series will be padded to
+     * the length of the longest time series.<br>
+     * Note 2: When ALIGN_START or ALIGN_END are used, the DataSet masking functionality is used. Thus, the returned DataSets
+     * will have the input and mask arrays set. These mask arrays identify whether an input/label is actually present,
+     * or whether the value is merely masked.<br>
+     */
+    public enum AlignmentMode {
+        EQUAL_LENGTH, ALIGN_START, ALIGN_END
+    }
+
+    private static final String READER_KEY = "reader";
+    private static final String READER_KEY_LABEL = "reader_labels";
+
+    private SequenceRecordReader recordReader;
+    private SequenceRecordReader labelsReader;
+    private int miniBatchSize = 10;
+    private final boolean regression;
+    private int labelIndex = -1;
+    private final int numPossibleLabels;
+    private int cursor = 0;
+    private int inputColumns = -1;
+    private int totalOutcomes = -1;
+    private boolean useStored = false;
+    private DataSet stored = null;
+    @Getter
+    private DataSetPreProcessor preProcessor;
+    private AlignmentMode alignmentMode;
+
+    private final boolean singleSequenceReaderMode;
+
+    @Getter
+    @Setter
+    private boolean collectMetaData = false;
+
+    private RecordReaderMultiDataSetIterator underlying;
+    private boolean underlyingIsDisjoint;
+
+    /**
+     * Constructor where features and labels come from different RecordReaders (for example, different files),
+     * and labels are for classification.
+     *
+     * @param featuresReader       SequenceRecordReader for the features
+     * @param labels               Labels: assume single value per time step, where values are integers in the range 0 to numPossibleLables-1
+     * @param miniBatchSize        Minibatch size for each call of next()
+     * @param numPossibleLabels    Number of classes for the labels
+     */
+    public SequenceRecordReaderDataSetIterator(SequenceRecordReader featuresReader, SequenceRecordReader labels,
+                    int miniBatchSize, int numPossibleLabels) {
+        this(featuresReader, labels, miniBatchSize, numPossibleLabels, false);
+    }
+
+    /**
+     * Constructor where features and labels come from different RecordReaders (for example, different files)
+     */
+    public SequenceRecordReaderDataSetIterator(SequenceRecordReader featuresReader, SequenceRecordReader labels,
+                    int miniBatchSize, int numPossibleLabels, boolean regression) {
+        this(featuresReader, labels, miniBatchSize, numPossibleLabels, regression, AlignmentMode.EQUAL_LENGTH);
+    }
+
+    /**
+     * Constructor where features and labels come from different RecordReaders (for example, different files)
+     */
+    public SequenceRecordReaderDataSetIterator(SequenceRecordReader featuresReader, SequenceRecordReader labels,
+                    int miniBatchSize, int numPossibleLabels, boolean regression, AlignmentMode alignmentMode) {
+        this.recordReader = featuresReader;
+        this.labelsReader = labels;
+        this.miniBatchSize = miniBatchSize;
+        this.numPossibleLabels = numPossibleLabels;
+        this.regression = regression;
+        this.alignmentMode = alignmentMode;
+        this.singleSequenceReaderMode = false;
+    }
+
+    /** Constructor where features and labels come from the SAME RecordReader (i.e., target/label is a column in the
+     * same data as the features). Defaults to regression = false - i.e., for classification
+     * @param reader SequenceRecordReader with data
+     * @param miniBatchSize size of each minibatch
+     * @param numPossibleLabels number of labels/classes for classification
+     * @param labelIndex index in input of the label index. If in regression mode and numPossibleLabels > 1, labelIndex denotes the
+     *                   first index for labels. Everything before that index will be treated as input(s) and
+     *                   everything from that index (inclusive) to the end will be treated as output(s)
+     */
+    public SequenceRecordReaderDataSetIterator(SequenceRecordReader reader, int miniBatchSize, int numPossibleLabels,
+                    int labelIndex) {
+        this(reader, miniBatchSize, numPossibleLabels, labelIndex, false);
+    }
+
+    /** Constructor where features and labels come from the SAME RecordReader (i.e., target/label is a column in the
+     * same data as the features)
+     * @param reader SequenceRecordReader with data
+     * @param miniBatchSize size of each minibatch
+     * @param numPossibleLabels number of labels/classes for classification
+     * @param labelIndex index in input of the label index. If in regression mode and numPossibleLabels > 1, labelIndex denotes the
+     *                   first index for labels. Everything before that index will be treated as input(s) and
+     *                   everything from that index (inclusive) to the end will be treated as output(s)
+     * @param regression Whether output is for regression or classification
+     */
+    public SequenceRecordReaderDataSetIterator(SequenceRecordReader reader, int miniBatchSize, int numPossibleLabels,
+                    int labelIndex, boolean regression) {
+        this.recordReader = reader;
+        this.labelsReader = null;
+        this.miniBatchSize = miniBatchSize;
+        this.regression = regression;
+        this.labelIndex = labelIndex;
+        this.numPossibleLabels = numPossibleLabels;
+        this.singleSequenceReaderMode = true;
+    }
+
+    private void initializeUnderlyingFromReader() {
+        initializeUnderlying(recordReader.nextSequence());
+        underlying.reset();
+    }
+
+    private void initializeUnderlying(SequenceRecord nextF) {
+        if (nextF.getSequenceRecord().isEmpty()) {
+            throw new ZeroLengthSequenceException();
+        }
+        int totalSizeF = nextF.getSequenceRecord().get(0).size();
+
+        //allow people to specify label index as -1 and infer the last possible label
+        if (singleSequenceReaderMode && numPossibleLabels >= 1 && labelIndex < 0) {
+            labelIndex = totalSizeF - 1;
+        } else if (!singleSequenceReaderMode && numPossibleLabels >= 1 && labelIndex < 0) {
+            labelIndex = 0;
+        }
+
+        recordReader.reset();
+
+        //Add readers
+        RecordReaderMultiDataSetIterator.Builder builder = new RecordReaderMultiDataSetIterator.Builder(miniBatchSize);
+        builder.addSequenceReader(READER_KEY, recordReader);
+        if (labelsReader != null) {
+            builder.addSequenceReader(READER_KEY_LABEL, labelsReader);
+        }
+
+
+        //Add outputs
+        if (singleSequenceReaderMode) {
+
+            if (labelIndex < 0 && numPossibleLabels < 0) {
+                //No labels - all values -> features array
+                builder.addInput(READER_KEY);
+            } else if (labelIndex == 0 || labelIndex == totalSizeF - 1) {  //Features: subset of columns
+                //Labels are first or last -> one input in underlying
+                int inputFrom;
+                int inputTo;
+                if (labelIndex < 0) {
+                    //No label
+                    inputFrom = 0;
+                    inputTo = totalSizeF - 1;
+                } else if (labelIndex == 0) {
+                    inputFrom = 1;
+                    inputTo = totalSizeF - 1;
+                } else {
+                    inputFrom = 0;
+                    inputTo = labelIndex - 1;
+                }
+
+                builder.addInput(READER_KEY, inputFrom, inputTo);
+
+                underlyingIsDisjoint = false;
+            } else if (regression && numPossibleLabels > 1){
+                //Multiple inputs and multiple outputs
+                int inputFrom = 0;
+                int inputTo = labelIndex - 1;
+                int outputFrom = labelIndex;
+                int outputTo = totalSizeF - 1;
+
+                builder.addInput(READER_KEY, inputFrom, inputTo);
+                builder.addOutput(READER_KEY, outputFrom, outputTo);
+
+                underlyingIsDisjoint = false;
+            } else {
+                //Multiple inputs (disjoint features case)
+                int firstFrom = 0;
+                int firstTo = labelIndex - 1;
+                int secondFrom = labelIndex + 1;
+                int secondTo = totalSizeF - 1;
+
+                builder.addInput(READER_KEY, firstFrom, firstTo);
+                builder.addInput(READER_KEY, secondFrom, secondTo);
+
+                underlyingIsDisjoint = true;
+            }
+
+            if(!(labelIndex < 0 && numPossibleLabels < 0)) {
+                if (regression && numPossibleLabels <= 1) {
+                    //Multiple output regression already handled
+                    builder.addOutput(READER_KEY, labelIndex, labelIndex);
+                } else if (!regression) {
+                    builder.addOutputOneHot(READER_KEY, labelIndex, numPossibleLabels);
+                }
+            }
+        } else {
+
+            //Features: entire reader
+            builder.addInput(READER_KEY);
+            underlyingIsDisjoint = false;
+
+            if (regression) {
+                builder.addOutput(READER_KEY_LABEL);
+            } else {
+                builder.addOutputOneHot(READER_KEY_LABEL, 0, numPossibleLabels);
+            }
+        }
+
+        if (alignmentMode != null) {
+            switch (alignmentMode) {
+                case EQUAL_LENGTH:
+                    builder.sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.EQUAL_LENGTH);
+                    break;
+                case ALIGN_START:
+                    builder.sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.ALIGN_START);
+                    break;
+                case ALIGN_END:
+                    builder.sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.ALIGN_END);
+                    break;
+            }
+        }
+
+        underlying = builder.build();
+
+        if (collectMetaData) {
+            underlying.setCollectMetaData(true);
+        }
+    }
+
+    private DataSet mdsToDataSet(MultiDataSet mds) {
+        INDArray f;
+        INDArray fm;
+        if (underlyingIsDisjoint) {
+            //Rare case: 2 input arrays -> concat
+            INDArray f1 = RecordReaderDataSetIterator.getOrNull(mds.getFeatures(), 0);
+            INDArray f2 = RecordReaderDataSetIterator.getOrNull(mds.getFeatures(), 1);
+            fm = RecordReaderDataSetIterator.getOrNull(mds.getFeaturesMaskArrays(), 0); //Per-example masking only on the input -> same for both
+
+            //Can assume 3d features here
+            f = Nd4j.createUninitialized(new long[] {f1.size(0), f1.size(1) + f2.size(1), f1.size(2)});
+            f.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.interval(0, f1.size(1)), NDArrayIndex.all()},
+                            f1);
+            f.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.interval(f1.size(1), f1.size(1) + f2.size(1)),
+                            NDArrayIndex.all()}, f2);
+        } else {
+            //Standard case
+            f = RecordReaderDataSetIterator.getOrNull(mds.getFeatures(), 0);
+            fm = RecordReaderDataSetIterator.getOrNull(mds.getFeaturesMaskArrays(), 0);
+        }
+
+        INDArray l = RecordReaderDataSetIterator.getOrNull(mds.getLabels(), 0);
+        INDArray lm = RecordReaderDataSetIterator.getOrNull(mds.getLabelsMaskArrays(), 0);
+
+        DataSet ds = new DataSet(f, l, fm, lm);
+
+        if (collectMetaData) {
+            List<Serializable> temp = mds.getExampleMetaData();
+            List<Serializable> temp2 = new ArrayList<>(temp.size());
+            for (Serializable s : temp) {
+                RecordMetaDataComposableMap m = (RecordMetaDataComposableMap) s;
+                if (singleSequenceReaderMode) {
+                    temp2.add(m.getMeta().get(READER_KEY));
+                } else {
+                    RecordMetaDataComposable c = new RecordMetaDataComposable(m.getMeta().get(READER_KEY),
+                                    m.getMeta().get(READER_KEY_LABEL));
+                    temp2.add(c);
+                }
+            }
+            ds.setExampleMetaData(temp2);
+        }
+
+        if (preProcessor != null) {
+            preProcessor.preProcess(ds);
+        }
+
+        return ds;
+    }
+
+    @Override
+    public boolean hasNext() {
+        if (underlying == null) {
+            initializeUnderlyingFromReader();
+        }
+        return underlying.hasNext();
+    }
+
+    @Override
+    public DataSet next() {
+        return next(miniBatchSize);
+    }
+
+
+    @Override
+    public DataSet next(int num) {
+        if (useStored) {
+            useStored = false;
+            DataSet temp = stored;
+            stored = null;
+            if (preProcessor != null)
+                preProcessor.preProcess(temp);
+            return temp;
+        }
+        if (!hasNext())
+            throw new NoSuchElementException();
+
+        if (underlying == null) {
+            initializeUnderlyingFromReader();
+        }
+
+        MultiDataSet mds = underlying.next(num);
+        DataSet ds = mdsToDataSet(mds);
+
+        if (totalOutcomes == -1) {
+            inputColumns = (int) ds.getFeatures().size(1);
+            totalOutcomes = ds.getLabels() == null ? -1 : (int) ds.getLabels().size(1);
+        }
+
+        return ds;
+    }
+
+    @Override
+    public int inputColumns() {
+        if (inputColumns != -1)
+            return inputColumns;
+        preLoad();
+        return inputColumns;
+    }
+
+    @Override
+    public int totalOutcomes() {
+        if (totalOutcomes != -1)
+            return totalOutcomes;
+        preLoad();
+        return totalOutcomes;
+    }
+
+    private void preLoad() {
+        stored = next();
+        useStored = true;
+
+        inputColumns = (int) stored.getFeatures().size(1);
+        totalOutcomes = (int) stored.getLabels().size(1);
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return true;
+    }
+
+    @Override
+    public void reset() {
+        if (underlying != null)
+            underlying.reset();
+
+        cursor = 0;
+        stored = null;
+        useStored = false;
+    }
+
+    @Override
+    public int batch() {
+        return miniBatchSize;
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        this.preProcessor = preProcessor;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException("Remove not supported for this iterator");
+    }
+
+    /**
+     * Load a single sequence example to a DataSet, using the provided RecordMetaData.
+     * Note that it is more efficient to load multiple instances at once, using {@link #loadFromMetaData(List)}
+     *
+     * @param recordMetaData RecordMetaData to load from. Should have been produced by the given record reader
+     * @return DataSet with the specified example
+     * @throws IOException If an error occurs during loading of the data
+     */
+    public DataSet loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return loadFromMetaData(Collections.singletonList(recordMetaData));
+    }
+
+    /**
+     * Load a multiple sequence examples to a DataSet, using the provided RecordMetaData instances.
+     *
+     * @param list List of RecordMetaData instances to load from. Should have been produced by the record reader provided
+     *             to the SequenceRecordReaderDataSetIterator constructor
+     * @return DataSet with the specified examples
+     * @throws IOException If an error occurs during loading of the data
+     */
+    public DataSet loadFromMetaData(List<RecordMetaData> list) throws IOException {
+        if (underlying == null) {
+            SequenceRecord r = recordReader.loadSequenceFromMetaData(list.get(0));
+            initializeUnderlying(r);
+        }
+
+        //Two cases: single vs. multiple reader...
+        List<RecordMetaData> l = new ArrayList<>(list.size());
+        if (singleSequenceReaderMode) {
+            for (RecordMetaData m : list) {
+                l.add(new RecordMetaDataComposableMap(Collections.singletonMap(READER_KEY, m)));
+            }
+        } else {
+            for (RecordMetaData m : list) {
+                RecordMetaDataComposable rmdc = (RecordMetaDataComposable) m;
+                Map<String, RecordMetaData> map = new HashMap<>(2);
+                map.put(READER_KEY, rmdc.getMeta()[0]);
+                map.put(READER_KEY_LABEL, rmdc.getMeta()[1]);
+                l.add(new RecordMetaDataComposableMap(map));
+            }
+        }
+
+        return mdsToDataSet(underlying.loadFromMetaData(l));
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/exception/ZeroLengthSequenceException.java

@@ -0,0 +1,31 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.datavec.exception;
+
+public class ZeroLengthSequenceException extends RuntimeException {
+    public ZeroLengthSequenceException() {
+        this("");
+    }
+
+    public ZeroLengthSequenceException(String type) {
+        super(String.format("Encountered zero-length %ssequence", type.equals("") ? "" : type + " "));
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java9/module-info.java

@@ -0,0 +1,9 @@
+open module deeplearning4j.datavec.iterators {
+    requires nd4j.common;
+    requires org.apache.commons.lang3;
+    requires slf4j.api;
+    requires datavec.api;
+    requires nd4j.api;
+    exports org.deeplearning4j.datasets.datavec;
+    exports org.deeplearning4j.datasets.datavec.exception;
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/pom.xml

@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ /* ******************************************************************************
+  ~  *
+  ~  *
+  ~  * This program and the accompanying materials are made available under the
+  ~  * terms of the Apache License, Version 2.0 which is available at
+  ~  * https://www.apache.org/licenses/LICENSE-2.0.
+  ~  *
+  ~  *  See the NOTICE file distributed with this work for additional
+  ~  *  information regarding copyright ownership.
+  ~  * Unless required by applicable law or agreed to in writing, software
+  ~  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  ~  * License for the specific language governing permissions and limitations
+  ~  * under the License.
+  ~  *
+  ~  * SPDX-License-Identifier: Apache-2.0
+  ~  ******************************************************************************/
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.eclipse.deeplearning4j</groupId>
+        <artifactId>deeplearning4j-data</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>deeplearning4j-utility-iterators</artifactId>
+    <packaging>jar</packaging>
+
+    <name>deeplearning4j-utility-iterators</name>
+
+    <properties>
+        <module.name>deeplearning4j.utility.iterators</module.name>
+    </properties>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.moditect</groupId>
+                <artifactId>moditect-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>nd4j-api</artifactId>
+        </dependency>
+    </dependencies>
+
+
+</project>

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AbstractDataSetIterator.java

@@ -0,0 +1,278 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.NonNull;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.common.primitives.Pair;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.LinkedBlockingQueue;
+
+public abstract class AbstractDataSetIterator<T> implements DataSetIterator {
+    private DataSetPreProcessor preProcessor;
+    private transient Iterable<Pair<T, T>> iterable;
+    private transient Iterator<Pair<T, T>> iterator;
+
+    private final int batchSize;
+
+    // FIXME: capacity 4 is triage here, proper investigation requires
+    private final LinkedBlockingQueue<DataSet> queue = new LinkedBlockingQueue<>(4);
+    private List<String> labels;
+    private int numFeatures = -1;
+    private int numLabels = -1;
+
+    protected AbstractDataSetIterator(@NonNull Iterable<Pair<T, T>> iterable, int batchSize) {
+        if (batchSize < 1)
+            throw new IllegalStateException("batchSize can't be < 1");
+
+        this.iterable = iterable;
+        this.iterator = this.iterable.iterator();
+        this.batchSize = batchSize;
+
+        fillQueue();
+    }
+
+
+    /**
+     * Like the standard next method but allows a
+     * customizable number of examples returned
+     *
+     * @param num the number of examples
+     * @return the next data applyTransformToDestination
+     */
+    @Override
+    public DataSet next(int num) {
+        throw new IllegalStateException("next(int) isn't supported for this DataSetIterator");
+    }
+
+    /**
+     * Input columns for the dataset
+     *
+     * @return
+     */
+    @Override
+    public int inputColumns() {
+        return numFeatures;
+    }
+
+    /**
+     * The number of labels for the dataset
+     *
+     * @return
+     */
+    @Override
+    public int totalOutcomes() {
+        return numLabels;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return iterable != null;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return true;
+    }
+
+    /**
+     * Resets the iterator back to the beginning
+     */
+    @Override
+    public void reset() {
+        queue.clear();
+        if (iterable != null)
+            iterator = iterable.iterator();
+    }
+
+    /**
+     * Batch size
+     *
+     * @return
+     */
+    @Override
+    public int batch() {
+        return batchSize;
+    }
+
+    /**
+     * Set a pre processor
+     *
+     * @param preProcessor a pre processor to set
+     */
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        this.preProcessor = preProcessor;
+    }
+
+    /**
+     * Get dataset iterator record reader labels
+     */
+    @Override
+    public List<String> getLabels() {
+        return labels;
+    }
+
+    /**
+     * Returns {@code true} if the iteration has more elements.
+     * (In other words, returns {@code true} if {@link #next} would
+     * return an element rather than throwing an exception.)
+     *
+     * @return {@code true} if the iteration has more elements
+     */
+    @Override
+    public boolean hasNext() {
+        fillQueue();
+        return !queue.isEmpty();
+    }
+
+    protected void fillQueue() {
+        if (queue.isEmpty()) {
+            List<INDArray> ndLabels = null;
+            List<INDArray> ndFeatures = null;
+            float[][] fLabels = null;
+            float[][] fFeatures = null;
+            double[][] dLabels = null;
+            double[][] dFeatures = null;
+
+            int sampleCount = 0;
+
+            for (int cnt = 0; cnt < batchSize; cnt++) {
+                if (iterator.hasNext()) {
+                    Pair<T, T> pair = iterator.next();
+                    if (numFeatures < 1) {
+                        if (pair.getFirst() instanceof INDArray) {
+                            numFeatures = (int) ((INDArray) pair.getFirst()).length();
+                            numLabels = (int) ((INDArray) pair.getSecond()).length();
+                        } else if (pair.getFirst() instanceof float[]) {
+                            numFeatures = ((float[]) pair.getFirst()).length;
+                            numLabels = ((float[]) pair.getSecond()).length;
+                        } else if (pair.getFirst() instanceof double[]) {
+                            numFeatures = ((double[]) pair.getFirst()).length;
+                            numLabels = ((double[]) pair.getSecond()).length;
+                        }
+                    }
+
+                    if (pair.getFirst() instanceof INDArray) {
+                        if (ndLabels == null) {
+                            ndLabels = new ArrayList<>();
+                            ndFeatures = new ArrayList<>();
+                        }
+                        ndFeatures.add(((INDArray) pair.getFirst()));
+                        ndLabels.add(((INDArray) pair.getSecond()));
+                    } else if (pair.getFirst() instanceof float[]) {
+                        if (fLabels == null) {
+                            fLabels = new float[batchSize][];
+                            fFeatures = new float[batchSize][];
+                        }
+                        fFeatures[sampleCount] = (float[]) pair.getFirst();
+                        fLabels[sampleCount] = (float[]) pair.getSecond();
+                    } else if (pair.getFirst() instanceof double[]) {
+                        if (dLabels == null) {
+                            dLabels = new double[batchSize][];
+                            dFeatures = new double[batchSize][];
+                        }
+                        dFeatures[sampleCount] = (double[]) pair.getFirst();
+                        dLabels[sampleCount] = (double[]) pair.getSecond();
+                    }
+
+                    sampleCount += 1;
+                } else
+                    break;
+            }
+
+            if (sampleCount == batchSize) {
+                INDArray labels = null;
+                INDArray features = null;
+                if (ndLabels != null) {
+                    labels = Nd4j.vstack(ndLabels);
+                    features = Nd4j.vstack(ndFeatures);
+                } else if (fLabels != null) {
+                    labels = Nd4j.create(fLabels);
+                    features = Nd4j.create(fFeatures);
+                } else if (dLabels != null) {
+                    labels = Nd4j.create(dLabels);
+                    features = Nd4j.create(dFeatures);
+                }
+
+                DataSet dataSet = new DataSet(features, labels);
+                try {
+                    queue.add(dataSet);
+                } catch (Exception e) {
+                    // live with it
+                }
+            }
+        }
+    }
+
+    /**
+     * Returns the next element in the iteration.
+     *
+     * @return the next element in the iteration
+     * @throws NoSuchElementException if the iteration has no more elements
+     */
+    @Override
+    public DataSet next() throws NoSuchElementException {
+        if (queue.isEmpty())
+            throw new NoSuchElementException();
+
+        DataSet dataSet = queue.poll();
+        if (preProcessor != null)
+            preProcessor.preProcess(dataSet);
+
+        return dataSet;
+    }
+
+    /**
+     * Removes from the underlying collection the last element returned
+     * by this iterator (optional operation).  This method can be called
+     * only once per call to {@link #next}.  The behavior of an iterator
+     * is unspecified if the underlying collection is modified while the
+     * iteration is in progress in any way other than by calling this
+     * method.
+     *
+     * @throws UnsupportedOperationException if the {@code remove}
+     *                                       operation is not supported by this iterator
+     * @throws IllegalStateException         if the {@code next} method has not
+     *                                       yet been called, or the {@code remove} method has already
+     *                                       been called after the last call to the {@code next}
+     *                                       method
+     * @implSpec The default implementation throws an instance of
+     * {@link UnsupportedOperationException} and performs no other action.
+     */
+    @Override
+    public void remove() {
+        // no-op
+    }
+
+    @Override
+    public DataSetPreProcessor getPreProcessor() {
+        return preProcessor;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncDataSetIterator.java

@@ -0,0 +1,79 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.extern.slf4j.Slf4j;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.dataset.callbacks.DataSetCallback;
+
+import java.util.concurrent.BlockingQueue;
+
+@Slf4j
+@Deprecated
+public class AsyncDataSetIterator extends org.nd4j.linalg.dataset.AsyncDataSetIterator {
+
+    /**
+     * Create an Async iterator with the default queue size of 8
+     * @param baseIterator Underlying iterator to wrap and fetch asynchronously from
+     */
+    public AsyncDataSetIterator(DataSetIterator baseIterator) {
+        super(baseIterator);
+    }
+
+    /**
+     * Create an Async iterator with the default queue size of 8
+     * @param iterator Underlying iterator to wrap and fetch asynchronously from
+     * @param queue    Queue size - number of iterators to
+     */
+    public AsyncDataSetIterator(DataSetIterator iterator, int queueSize, BlockingQueue<DataSet> queue) {
+        super(iterator, queueSize, queue);
+    }
+
+    public AsyncDataSetIterator(DataSetIterator baseIterator, int queueSize) {
+        super(baseIterator, queueSize);
+    }
+
+    public AsyncDataSetIterator(DataSetIterator baseIterator, int queueSize, boolean useWorkspace) {
+        super(baseIterator, queueSize, useWorkspace);
+    }
+
+    public AsyncDataSetIterator(DataSetIterator baseIterator, int queueSize, boolean useWorkspace, Integer deviceId) {
+        super(baseIterator, queueSize, useWorkspace, deviceId);
+    }
+
+    public AsyncDataSetIterator(DataSetIterator baseIterator, int queueSize, boolean useWorkspace, DataSetCallback callback) {
+        super(baseIterator, queueSize, useWorkspace, callback);
+    }
+
+    public AsyncDataSetIterator(DataSetIterator iterator, int queueSize, BlockingQueue<DataSet> queue, boolean useWorkspace) {
+        super(iterator, queueSize, queue, useWorkspace);
+    }
+
+    public AsyncDataSetIterator(DataSetIterator iterator, int queueSize, BlockingQueue<DataSet> queue, boolean useWorkspace, DataSetCallback callback) {
+        super(iterator, queueSize, queue, useWorkspace, callback);
+    }
+
+    public AsyncDataSetIterator(DataSetIterator iterator, int queueSize, BlockingQueue<DataSet> queue,
+                    boolean useWorkspace, DataSetCallback callback, Integer deviceId) {
+        super(iterator, queueSize, queue, useWorkspace, callback, deviceId);
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncMultiDataSetIterator.java

@@ -0,0 +1,69 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.extern.slf4j.Slf4j;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+import org.nd4j.linalg.dataset.callbacks.DataSetCallback;
+
+import java.util.concurrent.BlockingQueue;
+
+@Slf4j
+@Deprecated
+public class AsyncMultiDataSetIterator extends org.nd4j.linalg.dataset.AsyncMultiDataSetIterator {
+
+    public AsyncMultiDataSetIterator(MultiDataSetIterator baseIterator) {
+        super(baseIterator);
+    }
+
+    public AsyncMultiDataSetIterator(MultiDataSetIterator iterator, int queueSize, BlockingQueue<MultiDataSet> queue) {
+        super(iterator, queueSize, queue);
+    }
+
+    public AsyncMultiDataSetIterator(MultiDataSetIterator baseIterator, int queueSize) {
+        super(baseIterator, queueSize);
+    }
+
+    public AsyncMultiDataSetIterator(MultiDataSetIterator baseIterator, int queueSize, boolean useWorkspace) {
+        super(baseIterator, queueSize, useWorkspace);
+    }
+
+    public AsyncMultiDataSetIterator(MultiDataSetIterator baseIterator, int queueSize, boolean useWorkspace,
+                    Integer deviceId) {
+        super(baseIterator, queueSize, useWorkspace, deviceId);
+    }
+
+    public AsyncMultiDataSetIterator(MultiDataSetIterator iterator, int queueSize, BlockingQueue<MultiDataSet> queue,
+                    boolean useWorkspace) {
+        super(iterator, queueSize, queue, useWorkspace);
+    }
+
+    public AsyncMultiDataSetIterator(MultiDataSetIterator iterator, int queueSize, BlockingQueue<MultiDataSet> queue,
+                    boolean useWorkspace, DataSetCallback callback) {
+        super(iterator, queueSize, queue, useWorkspace, callback);
+    }
+
+    public AsyncMultiDataSetIterator(MultiDataSetIterator iterator, int queueSize, BlockingQueue<MultiDataSet> queue,
+                    boolean useWorkspace, DataSetCallback callback, Integer deviceId) {
+        super(iterator, queueSize, queue, useWorkspace, callback, deviceId);
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncShieldDataSetIterator.java

@@ -0,0 +1,186 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+
+import lombok.NonNull;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+
+import java.util.List;
+
+public class AsyncShieldDataSetIterator implements DataSetIterator {
+    private DataSetIterator backingIterator;
+
+    /**
+     * @param iterator Iterator to wrop, to disable asynchronous prefetching for
+     */
+    public AsyncShieldDataSetIterator(@NonNull DataSetIterator iterator) {
+        this.backingIterator = iterator;
+    }
+
+    /**
+     * Like the standard next method but allows a
+     * customizable number of examples returned
+     *
+     * @param num the number of examples
+     * @return the next data applyTransformToDestination
+     */
+    @Override
+    public DataSet next(int num) {
+        return backingIterator.next(num);
+    }
+
+    /**
+     * Input columns for the dataset
+     *
+     * @return
+     */
+    @Override
+    public int inputColumns() {
+        return backingIterator.inputColumns();
+    }
+
+    /**
+     * The number of labels for the dataset
+     *
+     * @return
+     */
+    @Override
+    public int totalOutcomes() {
+        return backingIterator.totalOutcomes();
+    }
+
+    /**
+     * Is resetting supported by this DataSetIterator? Many DataSetIterators do support resetting,
+     * but some don't
+     *
+     * @return true if reset method is supported; false otherwise
+     */
+    @Override
+    public boolean resetSupported() {
+        return backingIterator.resetSupported();
+    }
+
+    /**
+     * Does this DataSetIterator support asynchronous prefetching of multiple DataSet objects?
+     *
+     * PLEASE NOTE: This iterator ALWAYS returns FALSE
+     *
+     * @return true if asynchronous prefetching from this iterator is OK; false if asynchronous prefetching should not
+     * be used with this iterator
+     */
+    @Override
+    public boolean asyncSupported() {
+        return false;
+    }
+
+    /**
+     * Resets the iterator back to the beginning
+     */
+    @Override
+    public void reset() {
+        backingIterator.reset();
+    }
+
+    /**
+     * Batch size
+     *
+     * @return
+     */
+    @Override
+    public int batch() {
+        return backingIterator.batch();
+    }
+
+    /**
+     * Set a pre processor
+     *
+     * @param preProcessor a pre processor to set
+     */
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        backingIterator.setPreProcessor(preProcessor);
+    }
+
+    /**
+     * Returns preprocessors, if defined
+     *
+     * @return
+     */
+    @Override
+    public DataSetPreProcessor getPreProcessor() {
+        return backingIterator.getPreProcessor();
+    }
+
+    /**
+     * Get dataset iterator record reader labels
+     */
+    @Override
+    public List<String> getLabels() {
+        return backingIterator.getLabels();
+    }
+
+    /**
+     * Returns {@code true} if the iteration has more elements.
+     * (In other words, returns {@code true} if {@link #next} would
+     * return an element rather than throwing an exception.)
+     *
+     * @return {@code true} if the iteration has more elements
+     */
+    @Override
+    public boolean hasNext() {
+        return backingIterator.hasNext();
+    }
+
+    /**
+     * Returns the next element in the iteration.
+     *
+     * @return the next element in the iteration
+     */
+    @Override
+    public DataSet next() {
+        return backingIterator.next();
+    }
+
+    /**
+     * Removes from the underlying collection the last element returned
+     * by this iterator (optional operation).  This method can be called
+     * only once per call to {@link #next}.  The behavior of an iterator
+     * is unspecified if the underlying collection is modified while the
+     * iteration is in progress in any way other than by calling this
+     * method.
+     *
+     * @throws UnsupportedOperationException if the {@code remove}
+     *                                       operation is not supported by this iterator
+     * @throws IllegalStateException         if the {@code next} method has not
+     *                                       yet been called, or the {@code remove} method has already
+     *                                       been called after the last call to the {@code next}
+     *                                       method
+     * @implSpec The default implementation throws an instance of
+     * {@link UnsupportedOperationException} and performs no other action.
+     */
+    @Override
+    public void remove() {
+        // no-op
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncShieldMultiDataSetIterator.java

@@ -0,0 +1,136 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.NonNull;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+
+public class AsyncShieldMultiDataSetIterator implements MultiDataSetIterator {
+    private MultiDataSetIterator backingIterator;
+
+    public AsyncShieldMultiDataSetIterator(@NonNull MultiDataSetIterator iterator) {
+        this.backingIterator = iterator;
+    }
+
+    /**
+     * Fetch the next 'num' examples. Similar to the next method, but returns a specified number of examples
+     *
+     * @param num Number of examples to fetch
+     */
+    @Override
+    public MultiDataSet next(int num) {
+        return backingIterator.next(num);
+    }
+
+    /**
+     * Set the preprocessor to be applied to each MultiDataSet, before each MultiDataSet is returned.
+     *
+     * @param preProcessor MultiDataSetPreProcessor. May be null.
+     */
+    @Override
+    public void setPreProcessor(MultiDataSetPreProcessor preProcessor) {
+        backingIterator.setPreProcessor(preProcessor);
+    }
+
+    @Override
+    public MultiDataSetPreProcessor getPreProcessor() {
+        return backingIterator.getPreProcessor();
+    }
+
+    /**
+     * Is resetting supported by this DataSetIterator? Many DataSetIterators do support resetting,
+     * but some don't
+     *
+     * @return true if reset method is supported; false otherwise
+     */
+    @Override
+    public boolean resetSupported() {
+        return backingIterator.resetSupported();
+    }
+
+    /**
+     /**
+     * Does this DataSetIterator support asynchronous prefetching of multiple DataSet objects?
+     *
+     * PLEASE NOTE: This iterator ALWAYS returns FALSE
+     *
+     * @return true if asynchronous prefetching from this iterator is OK; false if asynchronous prefetching should not
+     * be used with this iterator
+     */
+    @Override
+    public boolean asyncSupported() {
+        return false;
+    }
+
+    /**
+     * Resets the iterator back to the beginning
+     */
+    @Override
+    public void reset() {
+        backingIterator.reset();
+    }
+
+    /**
+     * Returns {@code true} if the iteration has more elements.
+     * (In other words, returns {@code true} if {@link #next} would
+     * return an element rather than throwing an exception.)
+     *
+     * @return {@code true} if the iteration has more elements
+     */
+    @Override
+    public boolean hasNext() {
+        return backingIterator.hasNext();
+    }
+
+    /**
+     * Returns the next element in the iteration.
+     *
+     * @return the next element in the iteration
+     */
+    @Override
+    public MultiDataSet next() {
+        return backingIterator.next();
+    }
+
+    /**
+     * Removes from the underlying collection the last element returned
+     * by this iterator (optional operation).  This method can be called
+     * only once per call to {@link #next}.  The behavior of an iterator
+     * is unspecified if the underlying collection is modified while the
+     * iteration is in progress in any way other than by calling this
+     * method.
+     *
+     * @throws UnsupportedOperationException if the {@code remove}
+     *                                       operation is not supported by this iterator
+     * @throws IllegalStateException         if the {@code next} method has not
+     *                                       yet been called, or the {@code remove} method has already
+     *                                       been called after the last call to the {@code next}
+     *                                       method
+     * @implSpec The default implementation throws an instance of
+     * {@link UnsupportedOperationException} and performs no other action.
+     */
+    @Override
+    public void remove() {
+        // no-op
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/BaseDatasetIterator.java

@@ -0,0 +1,121 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.Getter;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.dataset.api.iterator.fetcher.BaseDataFetcher;
+
+import java.util.List;
+
+public class BaseDatasetIterator implements DataSetIterator {
+
+    protected int batch, numExamples;
+    protected BaseDataFetcher fetcher;
+    @Getter
+    protected DataSetPreProcessor preProcessor;
+
+
+    public BaseDatasetIterator(int batch, int numExamples, BaseDataFetcher fetcher) {
+        if(batch <= 0){
+            throw new IllegalArgumentException("Invalid minibatch size: must be > 0 (got: " + batch + ")");
+        }
+        this.batch = batch;
+        if (numExamples < 0)
+            numExamples = fetcher.totalExamples();
+
+        this.numExamples = numExamples;
+        this.fetcher = fetcher;
+    }
+
+    @Override
+    public boolean hasNext() {
+        return fetcher.hasMore() && fetcher.cursor() < numExamples;
+    }
+
+    @Override
+    public DataSet next() {
+        fetcher.fetch(batch);
+        DataSet result = fetcher.next();
+        if (preProcessor != null) {
+            preProcessor.preProcess(result);
+        }
+        return result;
+    }
+
+    @Override
+    public DataSet next(int num) {
+        fetcher.fetch(num);
+        DataSet next = fetcher.next();
+        if (preProcessor != null)
+            preProcessor.preProcess(next);
+        return next;
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int inputColumns() {
+        return fetcher.inputColumns();
+    }
+
+    @Override
+    public int totalOutcomes() {
+        return fetcher.totalOutcomes();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return true;
+    }
+
+    @Override
+    public void reset() {
+        fetcher.reset();
+    }
+
+    @Override
+    public int batch() {
+        return batch;
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        this.preProcessor = preProcessor;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/CombinedMultiDataSetPreProcessor.java

@@ -0,0 +1,76 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.NonNull;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class CombinedMultiDataSetPreProcessor implements MultiDataSetPreProcessor {
+
+    private List<MultiDataSetPreProcessor> preProcessors;
+
+    private CombinedMultiDataSetPreProcessor() {
+
+    }
+
+    @Override
+    public void preProcess(MultiDataSet multiDataSet) {
+        for (MultiDataSetPreProcessor preProcessor : preProcessors) {
+            preProcessor.preProcess(multiDataSet);
+        }
+    }
+
+    public static class Builder {
+        private List<MultiDataSetPreProcessor> preProcessors = new ArrayList<>();
+
+        public Builder() {
+
+        }
+
+        /**
+         * @param preProcessor to be added to list of preprocessors to be applied
+         */
+        public Builder addPreProcessor(@NonNull MultiDataSetPreProcessor preProcessor) {
+            preProcessors.add(preProcessor);
+            return this;
+        }
+
+        /**
+         * Inserts the specified preprocessor at the specified position to the list of preprocessors to be applied
+         * @param idx the position to apply the specified preprocessor at
+         * @param preProcessor to be added to list of preprocessors to be applied
+         */
+        public Builder addPreProcessor(int idx, @NonNull MultiDataSetPreProcessor preProcessor) {
+            preProcessors.add(idx, preProcessor);
+            return this;
+        }
+
+        public CombinedMultiDataSetPreProcessor build() {
+            CombinedMultiDataSetPreProcessor preProcessor = new CombinedMultiDataSetPreProcessor();
+            preProcessor.preProcessors = this.preProcessors;
+            return preProcessor;
+        }
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/CombinedPreProcessor.java

@@ -0,0 +1,73 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.NonNull;
+import org.nd4j.linalg.dataset.api.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class CombinedPreProcessor implements DataSetPreProcessor {
+    private List<DataSetPreProcessor> preProcessors;
+
+    private CombinedPreProcessor() {
+
+    }
+
+    /**
+     * Pre process a dataset sequentially
+     *
+     * @param toPreProcess the data set to pre process
+     */
+    @Override
+    public void preProcess(DataSet toPreProcess) {
+        for (DataSetPreProcessor preProcessor : preProcessors) {
+            preProcessor.preProcess(toPreProcess);
+        }
+    }
+
+    public static class Builder {
+        private List<DataSetPreProcessor> preProcessors = new ArrayList<>();
+
+        public Builder() {
+
+        }
+
+        public Builder addPreProcessor(@NonNull DataSetPreProcessor preProcessor) {
+            preProcessors.add(preProcessor);
+            return this;
+        }
+
+        public Builder addPreProcessor(int idx, @NonNull DataSetPreProcessor preProcessor) {
+            preProcessors.add(idx, preProcessor);
+            return this;
+        }
+
+
+        public CombinedPreProcessor build() {
+            CombinedPreProcessor preProcessor = new CombinedPreProcessor();
+            preProcessor.preProcessors = this.preProcessors;
+            return preProcessor;
+        }
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DataSetFetcher.java

@@ -0,0 +1,78 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import org.nd4j.linalg.dataset.DataSet;
+
+import java.io.Serializable;
+
+@Deprecated
+public interface DataSetFetcher extends Serializable {
+
+    /**
+     * Whether the dataset has more to load
+     * @return whether the data applyTransformToDestination has more to load
+     */
+    boolean hasMore();
+
+    /**
+     * Returns the next data applyTransformToDestination
+     * @return the next dataset
+     */
+    DataSet next();
+
+    /**
+     * Fetches the next dataset. You need to call this
+     * to get a new dataset, otherwise {@link #next()}
+     * just returns the last data applyTransformToDestination fetch
+     * @param numExamples the number of examples to fetch
+     */
+    void fetch(int numExamples);
+
+    /**
+     * The number of labels for a dataset
+     * @return the number of labels for a dataset
+     */
+    int totalOutcomes();
+
+    /**
+     * The length of a feature vector for an individual example
+     * @return the length of a feature vector for an individual example
+     */
+    int inputColumns();
+
+    /**
+     * The total number of examples
+     * @return the total number of examples
+     */
+    int totalExamples();
+
+    /**
+     * Returns the fetcher back to the beginning of the dataset
+     */
+    void reset();
+
+    /**
+     * Direct access to a number represenative of iterating through a dataset
+     * @return a cursor similar to an index
+     */
+    int cursor();
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DataSetIteratorSplitter.java

@@ -0,0 +1,344 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import lombok.val;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.exception.ND4JIllegalStateException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Slf4j
+public class DataSetIteratorSplitter {
+    protected DataSetIterator backedIterator;
+    protected final long totalExamples;
+    protected final double ratio;
+    protected final double[] ratios;
+    protected final long numTrain;
+    protected final long numTest;
+    protected final long numArbitrarySets;
+    protected final int[] splits;
+
+
+    protected AtomicLong counter = new AtomicLong(0);
+
+    protected AtomicBoolean resetPending = new AtomicBoolean(false);
+    protected DataSet firstTrain = null;
+
+
+    /**
+     * The only constructor
+     *
+     * @param baseIterator - iterator to be wrapped and split
+     * @param totalBatches - total batches in baseIterator
+     * @param ratio - train/test split ratio
+     */
+    public DataSetIteratorSplitter(@NonNull DataSetIterator baseIterator, long totalBatches, double ratio) {
+        if (!(ratio > 0.0 && ratio < 1.0))
+            throw new ND4JIllegalStateException("Ratio value should be in range of 0.0 > X < 1.0");
+
+        if (totalBatches < 0)
+            throw new ND4JIllegalStateException("totalExamples number should be positive value");
+
+        if (!baseIterator.resetSupported())
+            throw new ND4JIllegalStateException("Underlying iterator doesn't support reset, so it can't be used for runtime-split");
+
+
+        this.backedIterator = baseIterator;
+        this.totalExamples = totalBatches;
+        this.ratio = ratio;
+        ratios = new double[]{ratio, 1 - ratio};
+        this.numTrain = (long) (totalExamples * ratio);
+        this.numTest = totalExamples - numTrain;
+        this.numArbitrarySets = 2;
+        this.splits = new int[this.ratios.length];
+        for (int i = 0; i < this.splits.length; ++i) {
+            this.splits[i] = (int)(totalExamples * ratios[i]);
+        }
+
+
+        log.warn("IteratorSplitter is used: please ensure you don't use randomization/shuffle in underlying iterator!");
+    }
+
+    public DataSetIteratorSplitter(@NonNull DataSetIterator baseIterator, long totalBatches, double[] ratios) {
+        for (double ratio : ratios) {
+            if (!(ratio > 0.0 && ratio < 1.0))
+                throw new ND4JIllegalStateException("Ratio value should be in range of 0.0 > X < 1.0");
+        }
+
+        if (totalBatches < 0)
+            throw new ND4JIllegalStateException("totalExamples number should be positive value");
+
+        if (!baseIterator.resetSupported())
+            throw new ND4JIllegalStateException("Underlying iterator doesn't support reset, so it can't be used for runtime-split");
+
+
+        this.backedIterator = baseIterator;
+        this.totalExamples = totalBatches;
+        this.ratio = 0.0;
+        this.ratios = ratios;
+        this.numTrain = 0; //(long) (totalExamples * ratio);
+        this.numTest = 0; //totalExamples - numTrain;
+        this.numArbitrarySets = ratios.length;
+
+        this.splits = new int[this.ratios.length];
+        for (int i = 0; i < this.splits.length; ++i) {
+            this.splits[i] = (int)(totalExamples * ratios[i]);
+        }
+
+        log.warn("IteratorSplitter is used: please ensure you don't use randomization/shuffle in underlying iterator!");
+    }
+
+    public DataSetIteratorSplitter(@NonNull DataSetIterator baseIterator, int[] splits) {
+        int totalBatches = 0;
+        for (val v:splits)
+            totalBatches += v;
+
+        if (totalBatches < 0)
+            throw new ND4JIllegalStateException("totalExamples number should be positive value");
+
+        if (!baseIterator.resetSupported())
+            throw new ND4JIllegalStateException("Underlying iterator doesn't support reset, so it can't be used for runtime-split");
+
+
+        this.backedIterator = baseIterator;
+        this.totalExamples = totalBatches;
+        this.ratio = 0.0;
+        this.ratios = null;
+
+        this.numTrain = 0;
+        this.numTest = 0;
+        this.splits = splits;
+        this.numArbitrarySets = splits.length;
+
+        log.warn("IteratorSplitter is used: please ensure you don't use randomization/shuffle in underlying iterator!");
+    }
+
+    public List<DataSetIterator> getIterators() {
+        List<DataSetIterator> retVal = new ArrayList<>();
+        int partN = 0;
+        int bottom = 0;
+        for (final int split : splits) {
+                ScrollableDataSetIterator partIterator =
+                        new ScrollableDataSetIterator(partN++,
+                                backedIterator,
+                                counter,
+                                resetPending,
+                                firstTrain,
+                                new int[]{bottom,split});
+                bottom += split;
+                retVal.add(partIterator);
+        }
+        return retVal;
+    }
+
+
+    /**
+     * This method returns train iterator instance
+     *
+     * @return
+     */
+    @Deprecated
+    public DataSetIterator getTrainIterator() {
+        return new DataSetIterator() {
+            @Override
+            public DataSet next(int i) {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public List<String> getLabels() {
+                return backedIterator.getLabels();
+            }
+
+            @Override
+            public int inputColumns() {
+                return backedIterator.inputColumns();
+            }
+
+            @Override
+            public void remove() {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public int totalOutcomes() {
+                return backedIterator.totalOutcomes();
+            }
+
+            @Override
+            public boolean resetSupported() {
+                return backedIterator.resetSupported();
+            }
+
+            @Override
+            public boolean asyncSupported() {
+                return backedIterator.asyncSupported();
+            }
+
+            @Override
+            public void reset() {
+                resetPending.set(true);
+            }
+
+            @Override
+            public int batch() {
+                return backedIterator.batch();
+            }
+
+            @Override
+            public void setPreProcessor(DataSetPreProcessor dataSetPreProcessor) {
+                backedIterator.setPreProcessor(dataSetPreProcessor);
+            }
+
+            @Override
+            public DataSetPreProcessor getPreProcessor() {
+                return backedIterator.getPreProcessor();
+            }
+
+
+            @Override
+            public boolean hasNext() {
+                if (resetPending.get()) {
+                    if (resetSupported()) {
+                        backedIterator.reset();
+                        counter.set(0);
+                        resetPending.set(false);
+                    } else
+                        throw new UnsupportedOperationException("Reset isn't supported by underlying iterator");
+                }
+
+                val state = backedIterator.hasNext();
+                if (state && counter.get() < numTrain)
+                    return true;
+                else
+                    return false;
+            }
+
+            @Override
+            public DataSet next() {
+                counter.incrementAndGet();
+                val p = backedIterator.next();
+
+                if (counter.get() == 1 && firstTrain == null) {
+                    // first epoch ever, we'll save first dataset and will use it to check for equality later
+                    firstTrain =  p.copy();
+                    firstTrain.detach();
+                } else if (counter.get() == 1) {
+                    // epoch > 1, comparing first dataset to previously stored dataset. they should be equal
+                    int cnt = 0;
+                    if (!p.getFeatures().equalsWithEps(firstTrain.getFeatures(), 1e-5))
+                        throw new ND4JIllegalStateException("First examples do not match. Randomization was used?");
+                }
+
+                return p;
+            }
+        };
+    }
+
+    /**
+     * This method returns test iterator instance
+     *
+     * @return
+     */
+    @Deprecated
+    public DataSetIterator getTestIterator() {
+        return new DataSetIterator() {
+            @Override
+            public DataSet next(int i) {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public List<String> getLabels() {
+                return backedIterator.getLabels();
+            }
+
+            @Override
+            public int inputColumns() {
+                return backedIterator.inputColumns();
+            }
+
+            @Override
+            public void remove() {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public int totalOutcomes() {
+                return backedIterator.totalOutcomes();
+            }
+
+            @Override
+            public boolean resetSupported() {
+                return backedIterator.resetSupported();
+            }
+
+            @Override
+            public boolean asyncSupported() {
+                return backedIterator.asyncSupported();
+            }
+
+            @Override
+            public void reset() {
+                resetPending.set(true);
+            }
+
+            @Override
+            public int batch() {
+                return backedIterator.batch();
+            }
+
+            @Override
+            public void setPreProcessor(DataSetPreProcessor dataSetPreProcessor) {
+                backedIterator.setPreProcessor(dataSetPreProcessor);
+            }
+
+            @Override
+            public DataSetPreProcessor getPreProcessor() {
+                return backedIterator.getPreProcessor();
+            }
+
+
+            @Override
+            public boolean hasNext() {
+                val state = backedIterator.hasNext();
+                if (state && counter.get() < numTrain + numTest)
+                    return true;
+                else
+                    return false;
+            }
+
+            @Override
+            public DataSet next() {
+                counter.incrementAndGet();
+                return backedIterator.next();
+            }
+        };
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DoublesDataSetIterator.java

@@ -0,0 +1,35 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.NonNull;
+import org.nd4j.common.primitives.Pair;
+
+public class DoublesDataSetIterator extends AbstractDataSetIterator<double[]> {
+
+    /**
+     * @param iterable  Iterable to source data from
+     * @param batchSize Batch size for generated DataSet objects
+     */
+    public DoublesDataSetIterator(@NonNull Iterable<Pair<double[], double[]>> iterable, int batchSize) {
+        super(iterable, batchSize);
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyBlockDataSetIterator.java

@@ -0,0 +1,56 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import lombok.val;
+import org.nd4j.linalg.dataset.api.DataSet;
+import org.nd4j.linalg.dataset.api.iterator.BlockDataSetIterator;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+
+import java.util.ArrayList;
+
+@Slf4j
+public class DummyBlockDataSetIterator implements BlockDataSetIterator {
+    protected final DataSetIterator iterator;
+
+    public DummyBlockDataSetIterator(@NonNull DataSetIterator iterator) {
+        this.iterator = iterator;
+    }
+
+    @Override
+    public boolean hasAnything() {
+        return iterator.hasNext();
+    }
+
+    @Override
+    public DataSet[] next(int maxDatasets) {
+        val list = new ArrayList<DataSet>(maxDatasets);
+        int cnt = 0;
+        while (iterator.hasNext() && cnt < maxDatasets) {
+            list.add(iterator.next());
+            cnt++;
+        }
+
+        return list.toArray(new DataSet[list.size()]);
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyBlockMultiDataSetIterator.java

@@ -0,0 +1,59 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import lombok.val;
+import org.nd4j.linalg.dataset.api.DataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.iterator.BlockDataSetIterator;
+import org.nd4j.linalg.dataset.api.iterator.BlockMultiDataSetIterator;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+
+import java.util.ArrayList;
+
+@Slf4j
+public class DummyBlockMultiDataSetIterator implements BlockMultiDataSetIterator {
+    protected final MultiDataSetIterator iterator;
+
+    public DummyBlockMultiDataSetIterator(@NonNull MultiDataSetIterator iterator) {
+        this.iterator = iterator;
+    }
+
+    @Override
+    public boolean hasAnything() {
+        return iterator.hasNext();
+    }
+
+    @Override
+    public MultiDataSet[] next(int maxDatasets) {
+        val list = new ArrayList<MultiDataSet>(maxDatasets);
+        int cnt = 0;
+        while (iterator.hasNext() && cnt < maxDatasets) {
+            list.add(iterator.next());
+            cnt++;
+        }
+
+        return list.toArray(new MultiDataSet[list.size()]);
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyPreProcessor.java

@@ -0,0 +1,36 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import org.nd4j.linalg.dataset.api.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+
+public class DummyPreProcessor implements DataSetPreProcessor {
+    /**
+     * Pre process a dataset
+     *
+     * @param toPreProcess the data set to pre process
+     */
+    @Override
+    public void preProcess(DataSet toPreProcess) {
+        // no-op
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/EarlyTerminationDataSetIterator.java

@@ -0,0 +1,124 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+
+import java.util.List;
+
+public class EarlyTerminationDataSetIterator implements DataSetIterator {
+
+    private DataSetIterator underlyingIterator;
+    private int terminationPoint;
+    private int minibatchCount = 0;
+
+    /**
+     * Constructor takes the iterator to wrap and the number of minibatches after which the call to hasNext()
+     * will return false
+     * @param underlyingIterator, iterator to wrap
+     * @param terminationPoint, minibatches after which hasNext() will return false
+     */
+    public EarlyTerminationDataSetIterator(DataSetIterator underlyingIterator, int terminationPoint) {
+        if (terminationPoint <= 0)
+            throw new IllegalArgumentException(
+                            "Termination point (the number of calls to .next() or .next(num)) has to be > 0");
+        this.underlyingIterator = underlyingIterator;
+        this.terminationPoint = terminationPoint;
+    }
+
+    @Override
+    public DataSet next(int num) {
+        if (minibatchCount < terminationPoint) {
+            minibatchCount++;
+            return underlyingIterator.next(num);
+        } else {
+            throw new RuntimeException("Calls to next have exceeded termination point.");
+        }
+    }
+
+    @Override
+    public int inputColumns() {
+        return underlyingIterator.inputColumns();
+    }
+
+    @Override
+    public int totalOutcomes() {
+        return underlyingIterator.totalOutcomes();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return underlyingIterator.resetSupported();
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return underlyingIterator.asyncSupported();
+    }
+
+    @Override
+    public void reset() {
+        minibatchCount = 0;
+        underlyingIterator.reset();
+    }
+
+    @Override
+    public int batch() {
+        return underlyingIterator.batch();
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        underlyingIterator.setPreProcessor(preProcessor);
+    }
+
+    @Override
+    public DataSetPreProcessor getPreProcessor() {
+        return underlyingIterator.getPreProcessor();
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return underlyingIterator.getLabels();
+    }
+
+    @Override
+    public boolean hasNext() {
+        return underlyingIterator.hasNext() && minibatchCount < terminationPoint;
+    }
+
+    @Override
+    public DataSet next() {
+        if (minibatchCount < terminationPoint) {
+            minibatchCount++;
+            return underlyingIterator.next();
+        } else {
+            throw new RuntimeException("Calls to next have exceeded the allotted number of minibatches.");
+        }
+    }
+
+    @Override
+    public void remove() {
+        underlyingIterator.remove();
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/EarlyTerminationMultiDataSetIterator.java

@@ -0,0 +1,102 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+
+public class EarlyTerminationMultiDataSetIterator implements MultiDataSetIterator {
+
+    private MultiDataSetIterator underlyingIterator;
+    private int terminationPoint;
+    private int minibatchCount = 0;
+
+    /**
+     * Constructor takes the iterator to wrap and the number of minibatches after which the call to hasNext()
+     * will return false
+     * @param underlyingIterator, iterator to wrap
+     * @param terminationPoint, minibatches after which hasNext() will return false
+     */
+    public EarlyTerminationMultiDataSetIterator(MultiDataSetIterator underlyingIterator, int terminationPoint) {
+        if (terminationPoint <= 0)
+            throw new IllegalArgumentException(
+                            "Termination point (the number of calls to .next() or .next(num)) has to be > 0");
+        this.underlyingIterator = underlyingIterator;
+        this.terminationPoint = terminationPoint;
+    }
+
+    @Override
+    public MultiDataSet next(int num) {
+        if (minibatchCount < terminationPoint) {
+            minibatchCount++;
+            return underlyingIterator.next(num);
+        } else {
+            throw new RuntimeException("Calls to next have exceeded termination point.");
+        }
+    }
+
+    @Override
+    public void setPreProcessor(MultiDataSetPreProcessor preProcessor) {
+        underlyingIterator.setPreProcessor(preProcessor);
+    }
+
+    @Override
+    public MultiDataSetPreProcessor getPreProcessor() {
+        return underlyingIterator.getPreProcessor();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return underlyingIterator.resetSupported();
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return underlyingIterator.asyncSupported();
+    }
+
+    @Override
+    public void reset() {
+        minibatchCount = 0;
+        underlyingIterator.reset();
+    }
+
+    @Override
+    public boolean hasNext() {
+        return underlyingIterator.hasNext() && minibatchCount < terminationPoint;
+    }
+
+    @Override
+    public MultiDataSet next() {
+        if (minibatchCount < terminationPoint) {
+            minibatchCount++;
+            return underlyingIterator.next();
+        } else {
+            throw new RuntimeException("Calls to next have exceeded the allotted number of minibatches.");
+        }
+    }
+
+    @Override
+    public void remove() {
+        underlyingIterator.remove();
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ExistingDataSetIterator.java

@@ -0,0 +1,170 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+
+import lombok.Getter;
+import lombok.NonNull;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+
+import java.util.Iterator;
+import java.util.List;
+
+public class ExistingDataSetIterator implements DataSetIterator {
+    @Getter
+    private DataSetPreProcessor preProcessor;
+
+    private transient Iterable<DataSet> iterable;
+    private transient Iterator<DataSet> iterator;
+    private int totalExamples = 0;
+    private int numFeatures = 0;
+    private int numLabels = 0;
+    private List<String> labels;
+
+    /**
+     * Note that when using this constructor, resetting is not supported
+     * @param iterator Iterator to wrap
+     */
+    public ExistingDataSetIterator(@NonNull Iterator<DataSet> iterator) {
+        this.iterator = iterator;
+    }
+
+    /**
+     * Note that when using this constructor, resetting is not supported
+     * @param iterator Iterator to wrap
+     * @param labels   String labels. May be null.
+     */
+    public ExistingDataSetIterator(@NonNull Iterator<DataSet> iterator, @NonNull List<String> labels) {
+        this(iterator);
+        this.labels = labels;
+    }
+
+    /**
+     * Wraps the specified iterable. Supports resetting
+     * @param iterable Iterable to wrap
+     */
+    public ExistingDataSetIterator(@NonNull Iterable<DataSet> iterable) {
+        this.iterable = iterable;
+        this.iterator = iterable.iterator();
+    }
+
+    /**
+     * Wraps the specified iterable. Supports resetting
+     * @param iterable Iterable to wrap
+     * @param labels   Labels list. May be null
+     */
+    public ExistingDataSetIterator(@NonNull Iterable<DataSet> iterable, @NonNull List<String> labels) {
+        this(iterable);
+        this.labels = labels;
+    }
+
+
+    public ExistingDataSetIterator(@NonNull Iterable<DataSet> iterable, int totalExamples, int numFeatures,
+                    int numLabels) {
+        this(iterable);
+
+        this.totalExamples = totalExamples;
+        this.numFeatures = numFeatures;
+        this.numLabels = numLabels;
+    }
+
+    @Override
+    public DataSet next(int num) {
+        // TODO: this might be changed
+        throw new UnsupportedOperationException("next(int) isn't supported");
+    }
+
+    @Override
+    public int inputColumns() {
+        return numFeatures;
+    }
+
+    @Override
+    public int totalOutcomes() {
+        if (labels != null)
+            return labels.size();
+
+        return numLabels;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return iterable != null;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        //No need to asynchronously prefetch here: already in memory
+        return false;
+    }
+
+    @Override
+    public void reset() {
+        if (iterable != null)
+            this.iterator = iterable.iterator();
+        else
+            throw new IllegalStateException(
+                            "To use reset() method you need to provide Iterable<DataSet>, not Iterator");
+    }
+
+    @Override
+    public int batch() {
+        return 0;
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        this.preProcessor = preProcessor;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return labels;
+    }
+
+    @Override
+    public boolean hasNext() {
+        if (iterator != null)
+            return iterator.hasNext();
+
+        return false;
+    }
+
+    @Override
+    public DataSet next() {
+        if (preProcessor != null) {
+            DataSet ds = iterator.next();
+            if (!ds.isPreProcessed()) {
+                preProcessor.preProcess(ds);
+                ds.markAsPreProcessed();
+            }
+            return ds;
+        } else
+            return iterator.next();
+    }
+
+    @Override
+    public void remove() {
+        // no-op
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/FileSplitDataSetIterator.java

@@ -0,0 +1,129 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.datasets.iterator.callbacks.FileCallback;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+
+import java.io.File;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+@Slf4j
+public class FileSplitDataSetIterator implements DataSetIterator {
+    private DataSetPreProcessor preProcessor;
+
+    private List<File> files;
+    private int numFiles;
+    private AtomicInteger counter = new AtomicInteger(0);
+    private FileCallback callback;
+
+    /**
+     * @param files    List of files to iterate over
+     * @param callback Callback for loading the files
+     */
+    public FileSplitDataSetIterator(@NonNull List<File> files, @NonNull FileCallback callback) {
+        this.files = files;
+        this.numFiles = files.size();
+        this.callback = callback;
+    }
+
+
+    @Override
+    public DataSet next(int num) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int inputColumns() {
+        return 0;
+    }
+
+    @Override
+    public int totalOutcomes() {
+        return 0;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return true;
+    }
+
+    @Override
+    public void reset() {
+        counter.set(0);
+    }
+
+    @Override
+    public int batch() {
+        return 0;
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        this.preProcessor = preProcessor;
+    }
+
+    @Override
+    public DataSetPreProcessor getPreProcessor() {
+        return preProcessor;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+    @Override
+    public boolean hasNext() {
+        return counter.get() < numFiles;
+    }
+
+    @Override
+    public DataSet next() {
+        //        long time1 = System.nanoTime();
+        DataSet ds = callback.call(files.get(counter.getAndIncrement()));
+
+        if (preProcessor != null && ds != null)
+            preProcessor.preProcess(ds);
+
+        //        long time2 = System.nanoTime();
+
+        //        if (counter.get() % 5 == 0)
+        //            log.info("Device: [{}]; Time: [{}] ns;", Nd4j.getAffinityManager().getDeviceForCurrentThread(), time2 - time1);
+
+        return ds;
+    }
+
+    @Override
+    public void remove() {
+        // no-op
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/FloatsDataSetIterator.java

@@ -0,0 +1,35 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.NonNull;
+import org.nd4j.common.primitives.Pair;
+
+public class FloatsDataSetIterator extends AbstractDataSetIterator<float[]> {
+
+    /**
+     * @param iterable  Iterable to source data from
+     * @param batchSize Batch size for generated DataSet objects
+     */
+    public FloatsDataSetIterator(@NonNull Iterable<Pair<float[], float[]>> iterable, int batchSize) {
+        super(iterable, batchSize);
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/INDArrayDataSetIterator.java

@@ -0,0 +1,36 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.NonNull;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.common.primitives.Pair;
+
+public class INDArrayDataSetIterator extends AbstractDataSetIterator<INDArray> {
+
+    /**
+     * @param iterable  Iterable to source data from
+     * @param batchSize Batch size for generated DataSet objects
+     */
+    public INDArrayDataSetIterator(@NonNull Iterable<Pair<INDArray, INDArray>> iterable, int batchSize) {
+        super(iterable, batchSize);
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/IteratorDataSetIterator.java

@@ -0,0 +1,173 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+
+import lombok.Getter;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+
+import java.util.*;
+
+public class IteratorDataSetIterator implements DataSetIterator {
+
+    private final Iterator<DataSet> iterator;
+    private final int batchSize;
+    private final LinkedList<DataSet> queued; //Used when splitting larger examples than we want to return in a batch
+    @Getter
+    private DataSetPreProcessor preProcessor;
+
+    private int inputColumns = -1;
+    private int totalOutcomes = -1;
+
+    private int cursor = 0;
+
+    public IteratorDataSetIterator(Iterator<DataSet> iterator, int batchSize) {
+        this.iterator = iterator;
+        this.batchSize = batchSize;
+        this.queued = new LinkedList<>();
+    }
+
+    @Override
+    public boolean hasNext() {
+        return !queued.isEmpty() || iterator.hasNext();
+    }
+
+    @Override
+    public DataSet next() {
+        return next(batchSize);
+    }
+
+    @Override
+    public DataSet next(int num) {
+        if (!hasNext())
+            throw new NoSuchElementException();
+
+        List<DataSet> list = new ArrayList<>();
+        int countSoFar = 0;
+        while ((!queued.isEmpty() || iterator.hasNext()) && countSoFar < batchSize) {
+            DataSet next;
+            if (!queued.isEmpty()) {
+                next = queued.removeFirst();
+            } else {
+                next = iterator.next();
+            }
+            int nExamples = next.numExamples();
+            if (countSoFar + nExamples <= batchSize) {
+                //Add the entire DataSet as-is
+                list.add(next);
+            } else {
+                //Otherwise, split it
+                DataSet toKeep = (DataSet) next.getRange(0, batchSize - countSoFar);
+                DataSet toCache = (DataSet) next.getRange(batchSize - countSoFar, nExamples);
+                list.add(toKeep);
+                queued.add(toCache);
+            }
+
+            countSoFar += nExamples;
+        }
+
+        if (inputColumns == -1) {
+            //Set columns etc for later use
+            DataSet temp = list.get(0);
+
+            inputColumns = (int) temp.getFeatures().size(1);
+            totalOutcomes = temp.getLabels() == null ? 0 : (int) temp.getLabels().size(1); //May be null for layerwise pretraining
+        }
+
+        DataSet out;
+        if (list.size() == 1) {
+            out = list.get(0);
+        } else {
+            out = DataSet.merge(list);
+        }
+
+        if (preProcessor != null) {
+            if (!out.isPreProcessed()) {
+                preProcessor.preProcess(out);
+                out.markAsPreProcessed();
+            }
+        }
+        cursor += out.numExamples();
+        return out;
+    }
+
+    @Override
+    public int inputColumns() {
+        if (inputColumns != -1)
+            return inputColumns;
+        prefetchBatchSetInputOutputValues();
+        return inputColumns;
+    }
+
+    @Override
+    public int totalOutcomes() {
+        if (totalOutcomes != -1)
+            return totalOutcomes;
+        prefetchBatchSetInputOutputValues();
+        return totalOutcomes;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return false;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return true;
+    }
+
+    @Override
+    public void reset() {
+        throw new UnsupportedOperationException("Reset not supported");
+    }
+
+    @Override
+    public int batch() {
+        return batchSize;
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        this.preProcessor = preProcessor;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    private void prefetchBatchSetInputOutputValues() {
+        if (!iterator.hasNext())
+            return;
+        DataSet next = iterator.next();
+        inputColumns = (int) next.getFeatures().size(1);
+        totalOutcomes = (int) next.getLabels().size(1);
+        queued.add(next);
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/IteratorMultiDataSetIterator.java

@@ -0,0 +1,185 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+import org.nd4j.linalg.indexing.NDArrayIndex;
+
+import java.util.*;
+
+public class IteratorMultiDataSetIterator implements MultiDataSetIterator {
+
+    private final Iterator<MultiDataSet> iterator;
+    private final int batchSize;
+    private final LinkedList<MultiDataSet> queued; //Used when splitting larger examples than we want to return in a batch
+    private MultiDataSetPreProcessor preProcessor;
+
+    public IteratorMultiDataSetIterator(Iterator<MultiDataSet> iterator, int batchSize) {
+        this.iterator = iterator;
+        this.batchSize = batchSize;
+        this.queued = new LinkedList<>();
+    }
+
+    @Override
+    public boolean hasNext() {
+        return !queued.isEmpty() || iterator.hasNext();
+    }
+
+    @Override
+    public MultiDataSet next() {
+        return next(batchSize);
+    }
+
+    @Override
+    public MultiDataSet next(int num) {
+        if (!hasNext())
+            throw new NoSuchElementException();
+
+        List<MultiDataSet> list = new ArrayList<>();
+        int countSoFar = 0;
+        while ((!queued.isEmpty() || iterator.hasNext()) && countSoFar < batchSize) {
+            MultiDataSet next;
+            if (!queued.isEmpty()) {
+                next = queued.removeFirst();
+            } else {
+                next = iterator.next();
+            }
+
+            long nExamples = next.getFeatures(0).size(0);
+            if (countSoFar + nExamples <= batchSize) {
+                //Add the entire MultiDataSet as-is
+                list.add(next);
+            } else {
+                //Split the MultiDataSet
+
+                int nFeatures = next.numFeatureArrays();
+                int nLabels = next.numLabelsArrays();
+
+                INDArray[] fToKeep = new INDArray[nFeatures];
+                INDArray[] lToKeep = new INDArray[nLabels];
+                INDArray[] fToCache = new INDArray[nFeatures];
+                INDArray[] lToCache = new INDArray[nLabels];
+                INDArray[] fMaskToKeep = (next.getFeaturesMaskArrays() != null ? new INDArray[nFeatures] : null);
+                INDArray[] lMaskToKeep = (next.getLabelsMaskArrays() != null ? new INDArray[nLabels] : null);
+                INDArray[] fMaskToCache = (next.getFeaturesMaskArrays() != null ? new INDArray[nFeatures] : null);
+                INDArray[] lMaskToCache = (next.getLabelsMaskArrays() != null ? new INDArray[nLabels] : null);
+
+                for (int i = 0; i < nFeatures; i++) {
+                    INDArray fi = next.getFeatures(i);
+                    fToKeep[i] = getRange(fi, 0, batchSize - countSoFar);
+                    fToCache[i] = getRange(fi, batchSize - countSoFar, nExamples);
+
+                    if (fMaskToKeep != null) {
+                        INDArray fmi = next.getFeaturesMaskArray(i);
+                        fMaskToKeep[i] = getRange(fmi, 0, batchSize - countSoFar);
+                        fMaskToCache[i] = getRange(fmi, batchSize - countSoFar, nExamples);
+                    }
+                }
+
+                for (int i = 0; i < nLabels; i++) {
+                    INDArray li = next.getLabels(i);
+                    lToKeep[i] = getRange(li, 0, batchSize - countSoFar);
+                    lToCache[i] = getRange(li, batchSize - countSoFar, nExamples);
+
+                    if (lMaskToKeep != null) {
+                        INDArray lmi = next.getLabelsMaskArray(i);
+                        lMaskToKeep[i] = getRange(lmi, 0, batchSize - countSoFar);
+                        lMaskToCache[i] = getRange(lmi, batchSize - countSoFar, nExamples);
+                    }
+                }
+
+                MultiDataSet toKeep =
+                                new org.nd4j.linalg.dataset.MultiDataSet(fToKeep, lToKeep, fMaskToKeep, lMaskToKeep);
+                MultiDataSet toCache = new org.nd4j.linalg.dataset.MultiDataSet(fToCache, lToCache, fMaskToCache,
+                                lMaskToCache);
+                list.add(toKeep);
+                queued.add(toCache);
+            }
+
+            countSoFar += nExamples;
+        }
+
+        MultiDataSet out;
+        if (list.size() == 1) {
+            out = list.get(0);
+        } else {
+            out = org.nd4j.linalg.dataset.MultiDataSet.merge(list);
+        }
+
+        if (preProcessor != null)
+            preProcessor.preProcess(out);
+        return out;
+    }
+
+    private static INDArray getRange(INDArray arr, long exampleFrom, long exampleToExclusive) {
+        if (arr == null)
+            return null;
+
+        int rank = arr.rank();
+        switch (rank) {
+            case 2:
+                return arr.get(NDArrayIndex.interval(exampleFrom, exampleToExclusive), NDArrayIndex.all());
+            case 3:
+                return arr.get(NDArrayIndex.interval(exampleFrom, exampleToExclusive), NDArrayIndex.all(),
+                                NDArrayIndex.all());
+            case 4:
+                return arr.get(NDArrayIndex.interval(exampleFrom, exampleToExclusive), NDArrayIndex.all(),
+                                NDArrayIndex.all(), NDArrayIndex.all());
+            default:
+                throw new RuntimeException("Invalid rank: " + rank);
+        }
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return false;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        //No need to asynchronously prefetch here: already in memory
+        return false;
+    }
+
+    @Override
+    public void reset() {
+        throw new UnsupportedOperationException("Reset not supported");
+    }
+
+    @Override
+    public void setPreProcessor(MultiDataSetPreProcessor preProcessor) {
+        this.preProcessor = preProcessor;
+    }
+
+    @Override
+    public MultiDataSetPreProcessor getPreProcessor() {
+        return preProcessor;
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException("Not supported");
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/JointMultiDataSetIterator.java

@@ -0,0 +1,233 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.AllArgsConstructor;
+import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import lombok.val;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+
+@Slf4j
+@NoArgsConstructor
+@AllArgsConstructor
+public class JointMultiDataSetIterator implements MultiDataSetIterator {
+    protected MultiDataSetPreProcessor preProcessor;
+    protected Collection<DataSetIterator> iterators;
+    protected int outcome = -1;
+
+    /**
+     * @param iterators Underlying iterators to wrap
+     */
+    public JointMultiDataSetIterator(DataSetIterator... iterators) {
+        this.iterators = new ArrayList<DataSetIterator>();
+        this.iterators.addAll(Arrays.asList(iterators));
+        this.outcome = -1;
+    }
+
+    /**
+     *
+     * @param outcome   Index to get the label from. If < 0, labels from all iterators will be used to create the
+     *                  final MultiDataSet
+     * @param iterators Underlying iterators to wrap
+     */
+    public JointMultiDataSetIterator(int outcome, DataSetIterator... iterators){
+        this(iterators);
+        this.outcome = outcome;
+    }
+
+    /**
+     * Fetch the next 'num' examples. Similar to the next method, but returns a specified number of examples
+     *
+     * @param num Number of examples to fetch
+     */
+    @Override
+    public MultiDataSet next(int num) {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Set the preprocessor to be applied to each MultiDataSet, before each MultiDataSet is returned.
+     *
+     * @param preProcessor MultiDataSetPreProcessor. May be null.
+     */
+    @Override
+    public void setPreProcessor(MultiDataSetPreProcessor preProcessor) {
+        this.preProcessor = preProcessor;
+    }
+
+    /**
+     * Get the {@link MultiDataSetPreProcessor}, if one has previously been set.
+     * Returns null if no preprocessor has been set
+     *
+     * @return Preprocessor
+     */
+    @Override
+    public MultiDataSetPreProcessor getPreProcessor() {
+        return preProcessor;
+    }
+
+    /**
+     * Is resetting supported by this DataSetIterator? Many DataSetIterators do support resetting,
+     * but some don't
+     *
+     * @return true if reset method is supported; false otherwise
+     */
+    @Override
+    public boolean resetSupported() {
+        boolean sup = true;
+
+        for (val i: iterators)
+            if (!i.resetSupported()) {
+                sup = false;
+                break;
+            }
+
+        return sup;
+    }
+
+    /**
+     * Does this MultiDataSetIterator support asynchronous prefetching of multiple MultiDataSet objects?
+     * Most MultiDataSetIterators do, but in some cases it may not make sense to wrap this iterator in an
+     * iterator that does asynchronous prefetching. For example, it would not make sense to use asynchronous
+     * prefetching for the following types of iterators:
+     * (a) Iterators that store their full contents in memory already
+     * (b) Iterators that re-use features/labels arrays (as future next() calls will overwrite past contents)
+     * (c) Iterators that already implement some level of asynchronous prefetching
+     * (d) Iterators that may return different data depending on when the next() method is called
+     *
+     * @return true if asynchronous prefetching from this iterator is OK; false if asynchronous prefetching should not
+     * be used with this iterator
+     */
+    @Override
+    public boolean asyncSupported() {
+        boolean sup = true;
+
+        for (val i: iterators)
+            if (!i.asyncSupported()) {
+                sup = false;
+                break;
+            }
+
+        return sup;
+    }
+
+    /**
+     * Resets the iterator back to the beginning
+     */
+    @Override
+    public void reset() {
+        for (val i: iterators)
+            i.reset();
+    }
+
+    /**
+     * Returns {@code true} if the iteration has more elements.
+     * (In other words, returns {@code true} if {@link #next} would
+     * return an element rather than throwing an exception.)
+     *
+     * @return {@code true} if the iteration has more elements
+     */
+    @Override
+    public boolean hasNext() {
+        boolean has = true;
+
+        for (val i: iterators)
+            if (!i.hasNext()) {
+                has = false;
+                break;
+            }
+
+        return has;
+    }
+
+    /**
+     * Returns the next element in the iteration.
+     *
+     * @return the next element in the iteration
+     */
+    @Override
+    public MultiDataSet next() {
+        val features = new ArrayList<INDArray>();
+        val labels = new ArrayList<INDArray>();
+        val featuresMask = new ArrayList<INDArray>();
+        val labelsMask = new ArrayList<INDArray>();
+
+        boolean hasFM = false;
+        boolean hasLM = false;
+
+        int cnt = 0;
+        for (val i: iterators) {
+            val ds = i.next();
+
+            features.add(ds.getFeatures());
+            featuresMask.add(ds.getFeaturesMaskArray());
+
+            if (outcome < 0 || cnt == outcome) {
+                labels.add(ds.getLabels());
+                labelsMask.add(ds.getLabelsMaskArray());
+            }
+
+            if (ds.getFeaturesMaskArray() != null)
+                hasFM = true;
+
+            if (ds.getLabelsMaskArray() != null)
+                hasLM = true;
+
+            cnt++;
+        }
+
+        INDArray[] fm = hasFM ? featuresMask.toArray(new INDArray[0]) : null;
+        INDArray[] lm = hasLM ? labelsMask.toArray(new INDArray[0]) : null;
+
+        val mds = new org.nd4j.linalg.dataset.MultiDataSet(features.toArray(new INDArray[0]), labels.toArray(new INDArray[0]), fm, lm);
+
+        if (preProcessor != null)
+            preProcessor.preProcess(mds);
+
+        return mds;
+    }
+
+    /**
+     * PLEASE NOTE: This method is NOT implemented
+     *
+     * @throws UnsupportedOperationException if the {@code remove}
+     *                                       operation is not supported by this iterator
+     * @throws IllegalStateException         if the {@code next} method has not
+     *                                       yet been called, or the {@code remove} method has already
+     *                                       been called after the last call to the {@code next}
+     *                                       method
+     * @implSpec The default implementation throws an instance of
+     * {@link UnsupportedOperationException} and performs no other action.
+     */
+    @Override
+    public void remove() {
+        // noopp
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultiDataSetIteratorSplitter.java

@@ -0,0 +1,292 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import lombok.val;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+import org.nd4j.linalg.exception.ND4JIllegalStateException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Slf4j
+public class MultiDataSetIteratorSplitter {
+    protected MultiDataSetIterator backedIterator;
+    protected final long totalExamples;
+    protected final double ratio;
+    protected final long numTrain;
+    protected final long numTest;
+    protected final double[] ratios;
+    protected final long numArbitrarySets;
+    protected final int[] splits;
+
+    protected AtomicLong counter = new AtomicLong(0);
+
+    protected AtomicBoolean resetPending = new AtomicBoolean(false);
+    protected org.nd4j.linalg.dataset.MultiDataSet firstTrain = null;
+
+    /**
+     *
+     * @param baseIterator
+     * @param totalBatches - total number of batches in underlying iterator. this value will be used to determine number of test/train batches
+     * @param ratio - this value will be used as splitter. should be between in range of 0.0 > X < 1.0. I.e. if value 0.7 is provided, then 70% of total examples will be used for training, and 30% of total examples will be used for testing
+     */
+    public MultiDataSetIteratorSplitter(@NonNull MultiDataSetIterator baseIterator, long totalBatches, double ratio) {
+        if (!(ratio > 0.0 && ratio < 1.0))
+            throw new ND4JIllegalStateException("Ratio value should be in range of 0.0 > X < 1.0");
+
+        if (totalBatches < 0)
+            throw new ND4JIllegalStateException("totalExamples number should be positive value");
+
+        if (!baseIterator.resetSupported())
+            throw new ND4JIllegalStateException("Underlying iterator doesn't support reset, so it can't be used for runtime-split");
+
+
+        this.backedIterator = baseIterator;
+        this.totalExamples = totalBatches;
+        this.ratio = ratio;
+        this.numTrain = (long) (totalExamples * ratio);
+        this.numTest = totalExamples - numTrain;
+        this.ratios = null;
+        this.numArbitrarySets = 0;
+        this.splits = null;
+
+        log.warn("IteratorSplitter is used: please ensure you don't use randomization/shuffle in underlying iterator!");
+    }
+
+    public MultiDataSetIteratorSplitter(@NonNull MultiDataSetIterator baseIterator, long totalBatches, double[] ratios) {
+        for (double ratio : ratios) {
+            if (!(ratio > 0.0 && ratio < 1.0))
+                throw new ND4JIllegalStateException("Ratio value should be in range of 0.0 > X < 1.0");
+        }
+
+        if (totalBatches < 0)
+            throw new ND4JIllegalStateException("totalExamples number should be positive value");
+
+        if (!baseIterator.resetSupported())
+            throw new ND4JIllegalStateException("Underlying iterator doesn't support reset, so it can't be used for runtime-split");
+
+
+        this.backedIterator = baseIterator;
+        this.totalExamples = totalBatches;
+        this.ratio = 0.0;
+        this.numTrain = (long) (totalExamples * ratio);
+        this.numTest = totalExamples - numTrain;
+        this.ratios = null;
+        this.numArbitrarySets = ratios.length;
+
+        this.splits = new int[this.ratios.length];
+        for (int i = 0; i < this.splits.length; ++i) {
+            this.splits[i] = (int)(totalExamples * ratios[i]);
+        }
+
+        log.warn("IteratorSplitter is used: please ensure you don't use randomization/shuffle in underlying iterator!");
+    }
+
+    public MultiDataSetIteratorSplitter(@NonNull MultiDataSetIterator baseIterator, int[] splits) {
+
+        int totalBatches = 0;
+        for (val v:splits)
+            totalBatches += v;
+
+        if (totalBatches < 0)
+            throw new ND4JIllegalStateException("totalExamples number should be positive value");
+
+        if (!baseIterator.resetSupported())
+            throw new ND4JIllegalStateException("Underlying iterator doesn't support reset, so it can't be used for runtime-split");
+
+
+        this.backedIterator = baseIterator;
+        this.totalExamples = totalBatches;
+        this.ratio = 0.0;
+        this.numTrain = (long) (totalExamples * ratio);
+        this.numTest = totalExamples - numTrain;
+        this.ratios = null;
+        this.numArbitrarySets = splits.length;
+        this.splits = splits;
+
+        log.warn("IteratorSplitter is used: please ensure you don't use randomization/shuffle in underlying iterator!");
+    }
+
+    public List<MultiDataSetIterator> getIterators() {
+        List<MultiDataSetIterator> retVal = new ArrayList<>();
+        int partN = 0;
+        int bottom = 0;
+        for (final int split : splits) {
+            ScrollableMultiDataSetIterator partIterator =
+                    new ScrollableMultiDataSetIterator(partN++, backedIterator, counter, firstTrain,
+                            new int[]{bottom,split});
+            bottom += split;
+            retVal.add(partIterator);
+        }
+        return retVal;
+    }
+
+    /**
+     * This method returns train iterator instance
+     *
+     * @return
+     */
+    @Deprecated
+    public MultiDataSetIterator getTrainIterator() {
+        return new MultiDataSetIterator() {
+            @Override
+            public MultiDataSet next(int num) {
+                throw new UnsupportedOperationException("To be implemented yet");
+            }
+
+            @Override
+            public void setPreProcessor(MultiDataSetPreProcessor preProcessor) {
+                backedIterator.setPreProcessor(preProcessor);
+            }
+
+            @Override
+            public MultiDataSetPreProcessor getPreProcessor() {
+                return backedIterator.getPreProcessor();
+            }
+
+            @Override
+            public boolean resetSupported() {
+                return backedIterator.resetSupported();
+            }
+
+            @Override
+            public boolean asyncSupported() {
+                return backedIterator.asyncSupported();
+            }
+
+            @Override
+            public void reset() {
+                resetPending.set(true);
+            }
+
+            @Override
+            public boolean hasNext() {
+                if (resetPending.get()) {
+                    if (resetSupported()) {
+                        backedIterator.reset();
+                        counter.set(0);
+                        resetPending.set(false);
+                    } else
+                        throw new UnsupportedOperationException("Reset isn't supported by underlying iterator");
+                }
+
+                val state = backedIterator.hasNext();
+                if (state && counter.get() < numTrain)
+                    return true;
+                else
+                    return false;
+            }
+
+            @Override
+            public MultiDataSet next() {
+                counter.incrementAndGet();
+                val p = backedIterator.next();
+
+                if (counter.get() == 1 && firstTrain == null) {
+                    // first epoch ever, we'll save first dataset and will use it to check for equality later
+                    firstTrain = (org.nd4j.linalg.dataset.MultiDataSet) p.copy();
+                    firstTrain.detach();
+                } else if (counter.get() == 1) {
+                    // epoch > 1, comparing first dataset to previously stored dataset. they should be equal
+                    int cnt = 0;
+                    for (val c: p.getFeatures())
+                        if (!c.equalsWithEps(firstTrain.getFeatures()[cnt++], 1e-5))
+                            throw new ND4JIllegalStateException("First examples do not match. Randomization was used?");
+                }
+
+                return p;
+            }
+
+            @Override
+            public void remove() {
+                throw new UnsupportedOperationException();
+            }
+        };
+    }
+
+    /**
+     * This method returns test iterator instance
+     *
+     * @return
+     */
+    @Deprecated
+    public MultiDataSetIterator getTestIterator() {
+        return new MultiDataSetIterator() {
+            @Override
+            public MultiDataSet next(int num) {
+                throw new UnsupportedOperationException("To be implemented yet");
+            }
+
+            @Override
+            public void setPreProcessor(MultiDataSetPreProcessor preProcessor) {
+                backedIterator.setPreProcessor(preProcessor);
+            }
+
+            @Override
+            public MultiDataSetPreProcessor getPreProcessor() {
+                return backedIterator.getPreProcessor();
+            }
+
+            @Override
+            public boolean resetSupported() {
+                return backedIterator.resetSupported();
+            }
+
+            @Override
+            public boolean asyncSupported() {
+                return backedIterator.asyncSupported();
+            }
+
+
+            @Override
+            public void reset() {
+                resetPending.set(true);
+            }
+
+            @Override
+            public boolean hasNext() {
+                val state = backedIterator.hasNext();
+                if (state && counter.get() < numTrain + numTest)
+                    return true;
+                else
+                    return false;
+            }
+
+            @Override
+            public MultiDataSet next() {
+                counter.incrementAndGet();
+                return backedIterator.next();
+            }
+
+            @Override
+            public void remove() {
+                throw new UnsupportedOperationException();
+            }
+        };
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultiDataSetWrapperIterator.java

@@ -0,0 +1,122 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+
+import java.util.List;
+
+public class MultiDataSetWrapperIterator implements DataSetIterator {
+    protected MultiDataSetIterator iterator;
+    protected DataSetPreProcessor preProcessor;
+
+    /**
+     * @param iterator Undelying iterator to wrap
+     */
+    public MultiDataSetWrapperIterator(MultiDataSetIterator iterator) {
+        this.iterator = iterator;
+    }
+
+    @Override
+    public DataSet next(int num) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int inputColumns() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int totalOutcomes() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return iterator.resetSupported();
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return iterator.asyncSupported();
+    }
+
+    @Override
+    public void reset() {
+        iterator.reset();
+    }
+
+    @Override
+    public int batch() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        this.preProcessor = preProcessor;
+    }
+
+    @Override
+    public DataSetPreProcessor getPreProcessor() {
+        return preProcessor;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean hasNext() {
+        return iterator.hasNext();
+    }
+
+    @Override
+    public DataSet next() {
+        MultiDataSet mds = iterator.next();
+        if (mds.getFeatures().length > 1 || mds.getLabels().length > 1)
+            throw new UnsupportedOperationException(
+                            "This iterator is able to convert MultiDataSet with number of inputs/outputs of 1");
+
+        INDArray features = mds.getFeatures()[0];
+        INDArray labels = mds.getLabels() != null ? mds.getLabels()[0] : features;
+        INDArray fMask = mds.getFeaturesMaskArrays() != null ? mds.getFeaturesMaskArrays()[0] : null;
+        INDArray lMask = mds.getLabelsMaskArrays() != null ? mds.getLabelsMaskArrays()[0] : null;
+
+        DataSet ds = new DataSet(features, labels, fMask, lMask);
+
+        if (preProcessor != null)
+            preProcessor.preProcess(ds);
+
+        return ds;
+    }
+
+    @Override
+    public void remove() {
+        // no-op
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultipleEpochsIterator.java

@@ -0,0 +1,262 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.Setter;
+import org.nd4j.shade.guava.annotations.VisibleForTesting;
+import org.nd4j.shade.guava.collect.Lists;
+import lombok.Getter;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.atomic.AtomicLong;
+
+
+@Deprecated
+public class MultipleEpochsIterator implements DataSetIterator {
+    @VisibleForTesting
+    @Getter
+    @Setter
+    protected int epochs = 0;
+    protected int numEpochs;
+    protected int batch = 0;
+    protected int lastBatch = batch;
+    protected DataSetIterator iter;
+    protected DataSet ds;
+    protected List<DataSet> batchedDS = Lists.newArrayList();
+    protected static final Logger log = LoggerFactory.getLogger(MultipleEpochsIterator.class);
+    @Getter
+    protected DataSetPreProcessor preProcessor;
+    protected boolean newEpoch = false;
+    protected AtomicLong iterationsCounter = new AtomicLong(0);
+    protected long totalIterations = Long.MAX_VALUE;
+
+    @Deprecated
+    public MultipleEpochsIterator(int numEpochs, DataSetIterator iter) {
+        this.numEpochs = numEpochs;
+        this.iter = iter;
+    }
+
+    @Deprecated
+    public MultipleEpochsIterator(int numEpochs, DataSetIterator iter, int queueSize) {
+        this.numEpochs = numEpochs;
+        this.iter = iter;
+    }
+
+    @Deprecated
+    public MultipleEpochsIterator(DataSetIterator iter, int queueSize, long totalIterations) {
+        this.numEpochs = Integer.MAX_VALUE;
+        this.iter = iter;
+        this.totalIterations = totalIterations;
+    }
+
+    @Deprecated
+    public MultipleEpochsIterator(DataSetIterator iter, long totalIterations) {
+        this.numEpochs = Integer.MAX_VALUE;
+        this.iter = iter;
+        this.totalIterations = totalIterations;
+    }
+
+    @Deprecated
+    public MultipleEpochsIterator(int numEpochs, DataSet ds) {
+        this.numEpochs = numEpochs;
+        this.ds = ds;
+    }
+
+
+    /**
+     * Like the standard next method but allows a
+     * customizable number of examples returned
+     *
+     * @param num the number of examples
+     * @return the next data applyTransformToDestination
+     */
+    @Override
+    public DataSet next(int num) {
+        if (!hasNext()) {
+            throw new NoSuchElementException("No next element");
+        }
+        DataSet next;
+        batch++;
+        iterationsCounter.incrementAndGet();
+        if (iter == null) {
+            // return full DataSet
+            if (num == -1) {
+                next = ds;
+                if (epochs < numEpochs)
+                    trackEpochs();
+            }
+            // return DataSet broken into batches
+            else {
+                if (batchedDS.isEmpty() && num > 0)
+                    batchedDS = ds.batchBy(num);
+                next = batchedDS.get(batch);
+                if (batch + 1 == batchedDS.size()) {
+                    trackEpochs();
+                    if (epochs < numEpochs)
+                        batch = -1;
+                }
+            }
+        } else {
+            next = (num == -1 ? iter.next() : iter.next(num));
+            if (next == null) {
+                throw new IllegalStateException("Iterator returned null DataSet");
+            }
+            if (!iter.hasNext()) {
+                trackEpochs();
+                // track number of epochs and won't reset if it's over
+                if (epochs < numEpochs) {
+                    iter.reset();
+                    lastBatch = batch;
+                    batch = 0;
+                }
+            }
+        }
+        if (preProcessor != null)
+            preProcessor.preProcess(next);
+        return next;
+    }
+
+    public void trackEpochs() {
+        epochs++;
+        newEpoch = true;
+    }
+
+    @Override
+    public DataSet next() {
+        return next(-1);
+    }
+
+    /**
+     * Input columns for the dataset
+     *
+     * @return
+     */
+    @Override
+    public int inputColumns() {
+        return iter.inputColumns();
+    }
+
+    /**
+     * The number of labels for the dataset
+     *
+     * @return
+     */
+    @Override
+    public int totalOutcomes() {
+        return iter.totalOutcomes();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return iter.resetSupported();
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return iter.asyncSupported();
+    }
+
+    /**
+     * Resets the iterator back to the beginning
+     */
+    @Override
+    public void reset() {
+        if (!iter.resetSupported()) {
+            throw new IllegalStateException(
+                            "Cannot reset MultipleEpochsIterator with base iter that does not support reset");
+        }
+        epochs = 0;
+        lastBatch = batch;
+        batch = 0;
+        iterationsCounter.set(0);
+        iter.reset();
+    }
+
+    /**
+     * Batch size
+     *
+     * @return
+     */
+    @Override
+    public int batch() {
+        return iter.batch();
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        this.preProcessor = preProcessor;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return iter.getLabels();
+    }
+
+
+    /**
+     * Returns {@code true} if the iteration has more elements.
+     * (In other words, returns {@code true} if {@link #next} would
+     * return an element rather than throwing an exception.)
+     *
+     * @return {@code true} if the iteration has more elements
+     */
+    @Override
+    public boolean hasNext() {
+        if (iterationsCounter.get() >= totalIterations)
+            return false;
+
+        if (newEpoch) {
+            log.info("Epoch " + epochs + ", number of batches completed " + lastBatch);
+            newEpoch = false;
+        }
+        if (iter == null)
+            return (epochs < numEpochs) && ((!batchedDS.isEmpty() && batchedDS.size() > batch) || batchedDS.isEmpty());
+        else
+            // either there are still epochs to complete or its the first epoch
+            return (epochs < numEpochs) || (iter.hasNext() && (epochs == 0 || epochs == numEpochs));
+    }
+
+    /**
+     * Removes from the underlying collection the last element returned
+     * by this iterator (optional operation).  This method can be called
+     * only once per call to {@link #next}.  The behavior of an iterator
+     * is unspecified if the underlying collection is modified while the
+     * iteration is in progress in any way other than by calling this
+     * method.
+     *
+     * @throws UnsupportedOperationException if the {@code remove}
+     *                                       operation is not supported by this iterator
+     * @throws IllegalStateException         if the {@code next} method has not
+     *                                       yet been called, or the {@code remove} method has already
+     *                                       been called after the last call to the {@code next}
+     *                                       method
+     */
+    @Override
+    public void remove() {
+        iter.remove();
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/RandomDataSetIterator.java

@@ -0,0 +1,62 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import org.nd4j.linalg.factory.Nd4j;
+
+public class RandomDataSetIterator extends MultiDataSetWrapperIterator {
+
+    public enum Values {RANDOM_UNIFORM, RANDOM_NORMAL, ONE_HOT, ZEROS, ONES, BINARY, INTEGER_0_10, INTEGER_0_100, INTEGER_0_1000,
+        INTEGER_0_10000, INTEGER_0_100000;
+        public RandomMultiDataSetIterator.Values toMdsValues(){
+            return RandomMultiDataSetIterator.Values.valueOf(this.toString());
+        }
+    };
+
+    /**
+     * @param numMiniBatches Number of minibatches per epoch
+     * @param featuresShape  Features shape
+     * @param labelsShape    Labels shape
+     * @param featureValues  Type of values for the features
+     * @param labelValues    Type of values for the labels
+     */
+    public RandomDataSetIterator(int numMiniBatches, long[] featuresShape, long[] labelsShape, Values featureValues, Values labelValues){
+        this(numMiniBatches, featuresShape, labelsShape, featureValues, labelValues, Nd4j.order(), Nd4j.order());
+    }
+
+    /**
+     * @param numMiniBatches Number of minibatches per epoch
+     * @param featuresShape  Features shape
+     * @param labelsShape    Labels shape
+     * @param featureValues  Type of values for the features
+     * @param labelValues    Type of values for the labels
+     * @param featuresOrder  Array order ('c' or 'f') for the features array
+     * @param labelsOrder    Array order ('c' or 'f') for the labels array
+     */
+    public RandomDataSetIterator(int numMiniBatches, long[] featuresShape, long[] labelsShape, Values featureValues, Values labelValues,
+                                 char featuresOrder, char labelsOrder){
+        super(new RandomMultiDataSetIterator.Builder(numMiniBatches)
+                .addFeatures(featuresShape, featuresOrder, featureValues.toMdsValues())
+                .addLabels(labelsShape, labelsOrder, labelValues.toMdsValues())
+        .build());
+    }
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/RandomMultiDataSetIterator.java

@@ -0,0 +1,260 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.ops.transforms.Transforms;
+import org.nd4j.common.primitives.Triple;
+
+import java.util.*;
+
+public class RandomMultiDataSetIterator implements MultiDataSetIterator {
+
+    public enum Values {RANDOM_UNIFORM, RANDOM_NORMAL, ONE_HOT, ZEROS, ONES, BINARY, INTEGER_0_10, INTEGER_0_100, INTEGER_0_1000,
+        INTEGER_0_10000, INTEGER_0_100000}
+
+    private final int numMiniBatches;
+    private final List<Triple<long[], Character, Values>> features;
+    private final List<Triple<long[], Character, Values>> labels;
+    @Getter @Setter
+    private MultiDataSetPreProcessor preProcessor;
+
+    private int position;
+
+    /**
+     * @param numMiniBatches Number of minibatches per epoch
+     * @param features       Each triple in the list specifies the shape, array order and type of values for the features arrays
+     * @param labels         Each triple in the list specifies the shape, array order and type of values for the labels arrays
+     */
+    public RandomMultiDataSetIterator(int numMiniBatches, @NonNull List<Triple<long[], Character, Values>> features, @NonNull List<Triple<long[], Character, Values>> labels){
+        Preconditions.checkArgument(numMiniBatches > 0, "Number of minibatches must be positive: got %s", numMiniBatches);
+        Preconditions.checkArgument(features.size() > 0, "No features defined");
+        Preconditions.checkArgument(labels.size() > 0, "No labels defined");
+
+        this.numMiniBatches = numMiniBatches;
+        this.features = features;
+        this.labels = labels;
+    }
+
+    @Override
+    public MultiDataSet next(int i) {
+        return next();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return true;
+    }
+
+    @Override
+    public void reset() {
+        position = 0;
+    }
+
+    @Override
+    public boolean hasNext() {
+        return position < numMiniBatches;
+    }
+
+    @Override
+    public MultiDataSet next() {
+        if(!hasNext())
+            throw new NoSuchElementException("No next element");
+        INDArray[] f = new INDArray[features.size()];
+        INDArray[] l = new INDArray[labels.size()];
+
+        for( int i=0; i<f.length; i++ ){
+            Triple<long[], Character, Values> t = features.get(i);
+            f[i] = generate(t.getFirst(), t.getSecond(), t.getThird());
+        }
+
+        for( int i=0; i<l.length; i++ ){
+            Triple<long[], Character, Values> t = labels.get(i);
+            l[i] = generate(t.getFirst(), t.getSecond(), t.getThird());
+        }
+
+        position++;
+        MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(f,l);
+        if(preProcessor != null)
+            preProcessor.preProcess(mds);
+        return mds;
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    public static class Builder {
+
+        private int numMiniBatches;
+        private List<Triple<long[], Character, Values>> features = new ArrayList<>();
+        private List<Triple<long[], Character, Values>> labels = new ArrayList<>();
+
+        /**
+         * @param numMiniBatches Number of minibatches per epoch
+         */
+        public Builder(int numMiniBatches){
+            this.numMiniBatches = numMiniBatches;
+        }
+
+        /**
+         * Add a new features array to the iterator
+         * @param shape  Shape of the features
+         * @param values Values to fill the array with
+         */
+        public Builder addFeatures(long[] shape, Values values) {
+            return addFeatures(shape, 'c', values);
+        }
+
+        /**
+         * Add a new features array to the iterator
+         * @param shape  Shape of the features
+         * @param order  Order ('c' or 'f') for the array
+         * @param values Values to fill the array with
+         */
+        public Builder addFeatures(long[] shape, char order, Values values){
+            features.add(new Triple<>(shape, order, values));
+            return this;
+        }
+
+        /**
+         * Add a new labels array to the iterator
+         * @param shape  Shape of the features
+         * @param values Values to fill the array with
+         */
+        public Builder addLabels(long[] shape, Values values) {
+            return addLabels(shape, 'c', values);
+        }
+
+        /**
+         * Add a new labels array to the iterator
+         * @param shape  Shape of the features
+         * @param order  Order ('c' or 'f') for the array
+         * @param values Values to fill the array with
+         */
+        public Builder addLabels(long[] shape, char order, Values values){
+            labels.add(new Triple<>(shape, order, values));
+            return this;
+        }
+
+        public RandomMultiDataSetIterator build(){
+            return new RandomMultiDataSetIterator(numMiniBatches, features, labels);
+        }
+    }
+
+    /**
+     * Generate a random array with the specified shape
+     * @param shape  Shape of the array
+     * @param values Values to fill the array with
+     * @return Random array of specified shape + contents
+     */
+    public static INDArray generate(long[] shape, Values values) {
+        return generate(shape, Nd4j.order(), values);
+    }
+
+    /**
+     * Generate a random array with the specified shape and order
+     * @param shape  Shape of the array
+     * @param order  Order of array ('c' or 'f')
+     * @param values Values to fill the array with
+     * @return Random array of specified shape + contents
+     */
+    public static INDArray generate(long[] shape, char order, Values values){
+        switch (values){
+            case RANDOM_UNIFORM:
+                return Nd4j.rand(Nd4j.createUninitialized(shape,order));
+            case RANDOM_NORMAL:
+                return Nd4j.randn(Nd4j.createUninitialized(shape,order));
+            case ONE_HOT:
+                Random r = new Random(Nd4j.getRandom().nextLong());
+                INDArray out = Nd4j.create(shape,order);
+                if(shape.length == 1){
+                    out.putScalar(r.nextInt((int) shape[0]), 1.0);
+                } else if(shape.length == 2){
+                    for( int i=0; i<shape[0]; i++ ){
+                        out.putScalar(i, r.nextInt((int) shape[1]), 1.0);
+                    }
+                } else if(shape.length == 3){
+                    for( int i=0; i<shape[0]; i++ ){
+                        for(int j=0; j<shape[2]; j++ ){
+                            out.putScalar(i, r.nextInt((int) shape[1]), j, 1.0);
+                        }
+                    }
+                } else if(shape.length == 4){
+                    for( int i=0; i<shape[0]; i++ ){
+                        for(int j=0; j<shape[2]; j++ ){
+                            for(int k=0; k<shape[3]; k++ ) {
+                                out.putScalar(i, r.nextInt((int) shape[1]), j, k, 1.0);
+                            }
+                        }
+                    }
+                } else if(shape.length == 5){
+                    for( int i=0; i<shape[0]; i++ ){
+                        for(int j=0; j<shape[2]; j++ ){
+                            for(int k=0; k<shape[3]; k++ ) {
+                                for( int l=0; l<shape[4]; l++ ) {
+                                    out.putScalar(new int[]{i, r.nextInt((int) shape[1]), j, k, l}, 1.0);
+                                }
+                            }
+                        }
+                    }
+                } else {
+                    throw new RuntimeException("Not supported: rank 6+ arrays. Shape: " + Arrays.toString(shape));
+                }
+                return out;
+            case ZEROS:
+                return Nd4j.create(shape,order);
+            case ONES:
+                return Nd4j.createUninitialized(shape,order).assign(1.0);
+            case BINARY:
+                return Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(shape, order), 0.5));
+            case INTEGER_0_10:
+                return Transforms.floor(Nd4j.rand(shape).muli(10), false);
+            case INTEGER_0_100:
+                return Transforms.floor(Nd4j.rand(shape).muli(100), false);
+            case INTEGER_0_1000:
+                return Transforms.floor(Nd4j.rand(shape).muli(1000), false);
+            case INTEGER_0_10000:
+                return Transforms.floor(Nd4j.rand(shape).muli(10000), false);
+            case INTEGER_0_100000:
+                return Transforms.floor(Nd4j.rand(shape).muli(100000), false);
+            default:
+                throw new RuntimeException("Unknown enum value: " + values);
+
+        }
+    }
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ReconstructionDataSetIterator.java

@@ -0,0 +1,162 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+
+import lombok.Getter;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+
+import java.util.List;
+
+/**
+ * Wraps a data set iterator setting the first (feature matrix) as the labels.
+ *
+ * @author Adam Gibson
+ */
+public class ReconstructionDataSetIterator implements DataSetIterator {
+
+    private DataSetIterator iter;
+    @Getter
+    private DataSetPreProcessor preProcessor;
+
+    public ReconstructionDataSetIterator(DataSetIterator iter) {
+        this.iter = iter;
+    }
+
+    /**
+     * Like the standard next method but allows a
+     * customizable number of examples returned
+     *
+     * @param num the number of examples
+     * @return the next data applyTransformToDestination
+     */
+    @Override
+    public DataSet next(int num) {
+        DataSet ret = iter.next(num);
+        ret.setLabels(ret.getFeatures());
+        return ret;
+    }
+
+    /**
+     * Input columns for the dataset
+     *
+     * @return
+     */
+    @Override
+    public int inputColumns() {
+        return iter.inputColumns();
+    }
+
+    /**
+     * The number of labels for the dataset
+     *
+     * @return
+     */
+    @Override
+    public int totalOutcomes() {
+        return iter.totalOutcomes();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return iter.resetSupported();
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return iter.asyncSupported();
+    }
+
+    /**
+     * Resets the iterator back to the beginning
+     */
+    @Override
+    public void reset() {
+        iter.reset();
+    }
+
+    /**
+     * Batch size
+     *
+     * @return
+     */
+    @Override
+    public int batch() {
+        return iter.batch();
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        this.preProcessor = preProcessor;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+
+    /**
+     * Returns {@code true} if the iteration has more elements.
+     * (In other words, returns {@code true} if {@link #next} would
+     * return an element rather than throwing an exception.)
+     *
+     * @return {@code true} if the iteration has more elements
+     */
+    @Override
+    public boolean hasNext() {
+        return iter.hasNext();
+    }
+
+    /**
+     * Returns the next element in the iteration.
+     *
+     * @return the next element in the iteration
+     */
+    @Override
+    public DataSet next() {
+        DataSet next = iter.next();
+        next.setLabels(next.getFeatures());
+        return next;
+    }
+
+    /**
+     * Removes from the underlying collection the last element returned
+     * by this iterator (optional operation).  This method can be called
+     * only once per call to {@link #next}.  The behavior of an iterator
+     * is unspecified if the underlying collection is modified while the
+     * iteration is in progress in any way other than by calling this
+     * method.
+     *
+     * @throws UnsupportedOperationException if the {@code remove}
+     *                                       operation is not supported by this iterator
+     * @throws IllegalStateException         if the {@code next} method has not
+     *                                       yet been called, or the {@code remove} method has already
+     *                                       been called after the last call to the {@code next}
+     *                                       method
+     */
+    @Override
+    public void remove() {
+        iter.remove();
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/SamplingDataSetIterator.java

@@ -0,0 +1,34 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+
+import java.util.List;
+
+@Deprecated
+public class SamplingDataSetIterator extends org.nd4j.linalg.dataset.api.iterator.SamplingDataSetIterator {
+    public SamplingDataSetIterator(DataSet sampleFrom, int batchSize, int totalNumberSamples) {
+        super(sampleFrom, batchSize, totalNumberSamples);
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ScrollableDataSetIterator.java

@@ -0,0 +1,178 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.val;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.MultiDataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class ScrollableDataSetIterator implements DataSetIterator {
+    private int thisPart = 0;
+    private int top = 0;
+    private int bottom = 0;
+    protected DataSetIterator backedIterator;
+    protected AtomicLong counter = new AtomicLong(0);
+
+    protected AtomicBoolean resetPending = new AtomicBoolean(false);
+    protected DataSet firstTrain = null;
+    protected MultiDataSet firstMultiTrain = null;
+    private double ratio;
+    private long totalExamples;
+    private long itemsPerPart;
+    private long current;
+
+
+    public ScrollableDataSetIterator(int num, DataSetIterator backedIterator, AtomicLong counter,
+                                     AtomicBoolean resetPending, DataSet firstTrain, double ratio,
+                                     int totalExamples) {
+        this.thisPart = num;
+        this.backedIterator = backedIterator;
+        this.counter = counter;
+        this.resetPending = resetPending;
+        this.firstTrain = firstTrain;
+        this.ratio = ratio;
+        this.totalExamples = totalExamples;
+        this.itemsPerPart = (long)(totalExamples * ratio);
+        this.current = 0;
+    }
+
+    public ScrollableDataSetIterator(int num, DataSetIterator backedIterator, AtomicLong counter,
+                                     AtomicBoolean resetPending, DataSet firstTrain,
+                                     int[] itemsPerPart) {
+        this.thisPart = num;
+        this.bottom = itemsPerPart[0];
+        this.top = bottom + itemsPerPart[1];
+        this.itemsPerPart = top;
+
+        this.backedIterator = backedIterator;
+        this.counter = counter;
+        //this.resetPending = resetPending;
+        this.firstTrain = firstTrain;
+        //this.totalExamples = totalExamples;
+        this.current = 0;
+    }
+
+    @Override
+    public DataSet next(int i) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return backedIterator.getLabels();
+    }
+
+    @Override
+    public int inputColumns() {
+        return backedIterator.inputColumns();
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int totalOutcomes() {
+        return backedIterator.totalOutcomes();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return backedIterator.resetSupported();
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return backedIterator.asyncSupported();
+    }
+
+    @Override
+    public void reset() {
+        resetPending.set(true);
+    }
+
+    @Override
+    public int batch() {
+        return backedIterator.batch();
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor dataSetPreProcessor) {
+        backedIterator.setPreProcessor(dataSetPreProcessor);
+    }
+
+    @Override
+    public DataSetPreProcessor getPreProcessor() {
+
+        return backedIterator.getPreProcessor();
+    }
+
+
+    @Override
+    public boolean hasNext() {
+        if (resetPending.get()) {
+            if (resetSupported()) {
+                backedIterator.reset();
+                counter.set(0);
+                current = 0;
+                resetPending.set(false);
+            } else
+                throw new UnsupportedOperationException("Reset isn't supported by underlying iterator");
+        }
+
+        boolean state = false;
+        if (current >= top)
+            return false;
+        state = backedIterator.hasNext();
+        if (!state)
+            return false;
+        if (state && counter.get() < itemsPerPart)
+            return true;
+        else
+            return false;
+
+    }
+
+    @Override
+    public DataSet next() {
+        counter.incrementAndGet();
+        if ((current == 0) && (bottom != 0)) {
+            backedIterator.reset();
+            long cnt = current;
+            for (; cnt < bottom; ++cnt) {
+                if (backedIterator.hasNext())
+                    backedIterator.next();
+            }
+            current = cnt+1;
+        }
+        else current++;
+        val p = backedIterator.next();
+        return p;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ScrollableMultiDataSetIterator.java

@@ -0,0 +1,146 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.val;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+
+import javax.naming.OperationNotSupportedException;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class ScrollableMultiDataSetIterator implements MultiDataSetIterator {
+    private int thisPart = 0;
+    private int top = 0;
+    private int bottom = 0;
+    protected MultiDataSetIterator backedIterator;
+    protected AtomicLong counter = new AtomicLong(0);
+
+    protected AtomicBoolean resetPending = new AtomicBoolean(false);
+    protected DataSet firstTrain = null;
+    protected MultiDataSet firstMultiTrain = null;
+    private double ratio;
+    private long totalExamples;
+    private long itemsPerPart;
+    private long current;
+
+    public ScrollableMultiDataSetIterator(int num, MultiDataSetIterator backedIterator, AtomicLong counter,
+                                     MultiDataSet firstTrain,  int[] itemsPerPart) {
+        this.thisPart = num;
+        this.bottom = itemsPerPart[0];
+        this.top = bottom + itemsPerPart[1];
+        this.itemsPerPart = top;
+
+        this.counter = counter;
+        //this.resetPending = resetPending;
+        this.firstTrain = null;
+        this.firstMultiTrain = firstTrain;
+        //this.totalExamples = totalExamples;
+        this.current = 0;
+        this.backedIterator = backedIterator;
+        this.resetPending = resetPending;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return backedIterator.resetSupported();
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return backedIterator.asyncSupported();
+    }
+
+    @Override
+    public void reset() {
+        resetPending.set(true);
+    }
+
+    @Override
+    public void setPreProcessor(MultiDataSetPreProcessor dataSetPreProcessor) {
+        backedIterator.setPreProcessor(dataSetPreProcessor);
+    }
+
+    @Override
+    public MultiDataSetPreProcessor getPreProcessor() {
+
+        throw new UnsupportedOperationException();
+    }
+
+
+    @Override
+    public boolean hasNext() {
+        if (resetPending.get()) {
+            if (resetSupported()) {
+                backedIterator.reset();
+                counter.set(0);
+                current = 0;
+                resetPending.set(false);
+            } else
+                throw new UnsupportedOperationException("Reset isn't supported by underlying iterator");
+        }
+
+        boolean state = false;
+        if (current >= top)
+            return false;
+        state = backedIterator.hasNext();
+        if (!state)
+            return false;
+        if (state && counter.get() < itemsPerPart)
+            return true;
+        else
+            return false;
+
+    }
+
+    @Override
+    public MultiDataSet next() {
+        counter.incrementAndGet();
+        if ((current == 0) && (bottom != 0)) {
+            backedIterator.reset();
+            long cnt = current;
+            for (; cnt < bottom; ++cnt) {
+                if (backedIterator.hasNext())
+                    backedIterator.next();
+            }
+            current = cnt+1;
+        }
+        else current++;
+        val p = backedIterator.next();
+        return p;
+    }
+
+    @Override
+    public void remove() {
+        //
+    }
+
+    @Override
+    public MultiDataSet next(int i) {
+        throw new UnsupportedOperationException();
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/WorkspacesShieldDataSetIterator.java

@@ -0,0 +1,115 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator;
+
+import lombok.NonNull;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.factory.Nd4j;
+
+import java.util.List;
+
+public class WorkspacesShieldDataSetIterator implements DataSetIterator {
+    protected DataSetIterator iterator;
+
+    /**
+     * @param iterator The underlying iterator to detach values from
+     */
+    public WorkspacesShieldDataSetIterator(@NonNull DataSetIterator iterator) {
+        this.iterator = iterator;
+    }
+
+    @Override
+    public DataSet next(int num) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int inputColumns() {
+        return iterator.inputColumns();
+    }
+
+    @Override
+    public int totalOutcomes() {
+        return iterator.totalOutcomes();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return iterator.resetSupported();
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return iterator.asyncSupported();
+    }
+
+    @Override
+    public void reset() {
+        iterator.reset();
+    }
+
+    @Override
+    public int batch() {
+        return iterator.batch();
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        iterator.setPreProcessor(preProcessor);
+    }
+
+    @Override
+    public DataSetPreProcessor getPreProcessor() {
+        return iterator.getPreProcessor();
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return iterator.getLabels();
+    }
+
+    @Override
+    public boolean hasNext() {
+        return iterator.hasNext();
+    }
+
+    @Override
+    public DataSet next() {
+        DataSet ds = iterator.next();
+
+        if (ds.getFeatures().isAttached()) {
+            if (Nd4j.getMemoryManager().getCurrentWorkspace() == null) {
+                ds.detach();
+            } else {
+                ds.migrate();
+            }
+        }
+
+        return ds;
+    }
+
+    @Override
+    public void remove() {
+        // no-op
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/DataSetCallback.java

@@ -0,0 +1,30 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.callbacks;
+
+
+import org.nd4j.linalg.dataset.api.DataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+
+@Deprecated
+public interface DataSetCallback extends org.nd4j.linalg.dataset.callbacks.DataSetCallback {
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/DataSetDeserializer.java

@@ -0,0 +1,36 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.callbacks;
+
+import lombok.extern.slf4j.Slf4j;
+import org.nd4j.linalg.dataset.DataSet;
+
+import java.io.File;
+
+@Slf4j
+public class DataSetDeserializer implements FileCallback {
+    @Override
+    public <T> T call(File file) {
+        DataSet dataSet = new DataSet();
+        dataSet.load(file);
+        return (T) dataSet;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/DefaultCallback.java

@@ -0,0 +1,31 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.callbacks;
+
+import org.nd4j.linalg.api.concurrency.AffinityManager;
+import org.nd4j.linalg.dataset.api.DataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.factory.Nd4j;
+
+@Deprecated
+public class DefaultCallback extends org.nd4j.linalg.dataset.callbacks.DefaultCallback {
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/FileCallback.java

@@ -0,0 +1,28 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.callbacks;
+
+import java.io.File;
+
+public interface FileCallback {
+
+    <T> T call(File file);
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/InterleavedDataSetCallback.java

@@ -0,0 +1,106 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.callbacks;
+
+import lombok.extern.slf4j.Slf4j;
+import org.nd4j.linalg.api.memory.MemoryWorkspace;
+import org.nd4j.linalg.api.memory.conf.WorkspaceConfiguration;
+import org.nd4j.linalg.api.memory.enums.AllocationPolicy;
+import org.nd4j.linalg.api.memory.enums.LearningPolicy;
+import org.nd4j.linalg.api.memory.enums.ResetPolicy;
+import org.nd4j.linalg.api.memory.enums.SpillPolicy;
+import org.nd4j.linalg.dataset.api.DataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.factory.Nd4j;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Slf4j
+public class InterleavedDataSetCallback implements DataSetCallback {
+    private List<MemoryWorkspace> workspaces = new ArrayList<>();
+    private int bufferSize;
+    private int numWorkspaces;
+
+    private boolean isInitialized = false;
+
+    private AtomicLong counterInput = new AtomicLong(0);
+
+    public InterleavedDataSetCallback(int bufferSize) {
+        this.bufferSize = bufferSize;
+    }
+
+    protected void initializeWorkspaces(long size) {
+        WorkspaceConfiguration configuration = WorkspaceConfiguration.builder().initialSize(size)
+                        .overallocationLimit(bufferSize).policyReset(ResetPolicy.ENDOFBUFFER_REACHED)
+                        .policyAllocation(AllocationPolicy.OVERALLOCATE).policySpill(SpillPolicy.EXTERNAL)
+                        .policyLearning(LearningPolicy.NONE).build();
+
+        int numDevices = Nd4j.getAffinityManager().getNumberOfDevices();
+        int cDevice = Nd4j.getAffinityManager().getDeviceForCurrentThread();
+        for (int i = 0; i < numDevices; i++) {
+            Nd4j.getAffinityManager().unsafeSetDevice(i);
+            workspaces.add(Nd4j.getWorkspaceManager().createNewWorkspace(configuration, "IDSC-" + i, i));
+        }
+
+        Nd4j.getAffinityManager().unsafeSetDevice(cDevice);
+        numWorkspaces = numDevices;
+        isInitialized = true;
+    }
+
+    @Override
+    public void call(DataSet dataSet) {
+        if (!isInitialized)
+            initializeWorkspaces(dataSet.getMemoryFootprint());
+
+        Nd4j.getExecutioner().commit();
+
+        int currIdx = (int) (counterInput.getAndIncrement() % numWorkspaces);
+        MemoryWorkspace currWs = Nd4j.getMemoryManager().getCurrentWorkspace();
+        Nd4j.getMemoryManager().setCurrentWorkspace(workspaces.get(currIdx));
+
+        dataSet.migrate();
+
+        Nd4j.getMemoryManager().setCurrentWorkspace(currWs);
+    }
+
+    @Override
+    public void call(MultiDataSet multiDataSet) {
+        if (!isInitialized)
+            initializeWorkspaces(multiDataSet.getMemoryFootprint());
+
+        Nd4j.getExecutioner().commit();
+
+        int currIdx = (int) (counterInput.getAndIncrement() % numWorkspaces);
+        MemoryWorkspace currWs = Nd4j.getMemoryManager().getCurrentWorkspace();
+        Nd4j.getMemoryManager().setCurrentWorkspace(workspaces.get(currIdx));
+
+        multiDataSet.migrate();
+
+        Nd4j.getMemoryManager().setCurrentWorkspace(currWs);
+    }
+
+    @Override
+    public void reset() {
+        counterInput.set(0);
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/file/BaseFileIterator.java

@@ -0,0 +1,197 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.file;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+import org.apache.commons.io.FileUtils;
+import org.nd4j.linalg.api.memory.MemoryWorkspace;
+import org.nd4j.common.collection.CompactHeapStringList;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.common.util.MathUtils;
+
+import java.io.File;
+import java.util.*;
+
+public abstract class BaseFileIterator<T, P> implements Iterator<T> {
+
+    protected final List<String> list;
+    protected final int batchSize;
+    protected final Random rng;
+
+    protected int[] order;
+    protected int position;
+
+    private T partialStored;
+    @Getter
+    @Setter
+    protected P preProcessor;
+
+
+    protected BaseFileIterator(@NonNull File rootDir, int batchSize, String... validExtensions) {
+        this(new File[]{rootDir}, true, new Random(), batchSize, validExtensions);
+    }
+
+    protected BaseFileIterator(@NonNull File[] rootDirs, boolean recursive, Random rng, int batchSize, String... validExtensions) {
+        this.batchSize = batchSize;
+        this.rng = rng;
+
+        list = new CompactHeapStringList();
+        for(File rootDir : rootDirs) {
+            Collection<File> c = FileUtils.listFiles(rootDir, validExtensions, recursive);
+            if (c.isEmpty()) {
+                throw new IllegalStateException("Root directory is empty (no files found) " + (validExtensions != null ? " (or all files rejected by extension filter)" : ""));
+            }
+            for (File f : c) {
+                list.add(f.getPath());
+            }
+        }
+
+        if (rng != null) {
+            order = new int[list.size()];
+            for (int i = 0; i < order.length; i++) {
+                order[i] = i;
+            }
+            MathUtils.shuffleArray(order, rng);
+        }
+    }
+
+    @Override
+    public boolean hasNext() {
+        return partialStored != null || position < list.size();
+    }
+
+    @Override
+    public T next() {
+        if (!hasNext()) {
+            throw new NoSuchElementException("No next element");
+        }
+
+        T next;
+        if (partialStored != null) {
+            next = partialStored;
+            partialStored = null;
+        } else {
+            int nextIdx = (order != null ? order[position++] : position++);
+            next = load(new File(list.get(nextIdx)));
+        }
+        if (batchSize <= 0) {
+            //Don't recombine, return as-is
+            return next;
+        }
+
+        if (sizeOf(next) == batchSize) {
+            return next;
+        }
+
+        int exampleCount = 0;
+        List<T> toMerge = new ArrayList<>();
+        toMerge.add(next);
+        exampleCount += sizeOf(next);
+
+        while (exampleCount < batchSize && hasNext()) {
+            int nextIdx = (order != null ? order[position++] : position++);
+            next = load(new File(list.get(nextIdx)));
+            exampleCount += sizeOf(next);
+            toMerge.add(next);
+        }
+
+        T ret = mergeAndStoreRemainder(toMerge);
+        applyPreprocessor(ret);
+        return ret;
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    protected T mergeAndStoreRemainder(List<T> toMerge) {
+        //Could be smaller or larger
+        List<T> correctNum = new ArrayList<>();
+        List<T> remainder = new ArrayList<>();
+        int soFar = 0;
+        for (T t : toMerge) {
+            long size = sizeOf(t);
+
+            if (soFar + size <= batchSize) {
+                correctNum.add(t);
+                soFar += size;
+            } else if (soFar < batchSize) {
+                //Split and add some
+                List<T> split = split(t);
+                if (rng != null) {
+                    Collections.shuffle(split, rng);
+                }
+                for (T t2 : split) {
+                    if (soFar < batchSize) {
+                        correctNum.add(t2);
+                        soFar += sizeOf(t2);
+                    } else {
+                        remainder.add(t2);
+                    }
+                }
+            } else {
+                //Don't need any of this
+                remainder.add(t);
+            }
+        }
+
+        T ret = merge(correctNum);
+        if (remainder.isEmpty()) {
+            this.partialStored = null;
+        } else {
+            try (MemoryWorkspace ws = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) {
+                this.partialStored = merge(remainder);
+            }
+        }
+
+        return ret;
+    }
+
+
+    public void reset() {
+        position = 0;
+        if (rng != null) {
+            MathUtils.shuffleArray(order, rng);
+        }
+    }
+
+    public boolean resetSupported() {
+        return true;
+    }
+
+    public boolean asyncSupported() {
+        return true;
+    }
+
+
+    protected abstract T load(File f);
+
+    protected abstract long sizeOf(T of);
+
+    protected abstract List<T> split(T toSplit);
+
+    protected abstract T merge(List<T> toMerge);
+
+    protected abstract void applyPreprocessor(T toPreProcess);
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/file/FileDataSetIterator.java

@@ -0,0 +1,185 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.file;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+
+import java.io.File;
+import java.util.List;
+import java.util.Random;
+
+public class FileDataSetIterator extends BaseFileIterator<DataSet, DataSetPreProcessor> implements DataSetIterator {
+
+    @Getter
+    @Setter
+    private List<String> labels;
+
+    /**
+     * Create a FileDataSetIterator with the following default settings:<br>
+     * - Recursive: files in subdirectories are included<br>
+     * - Randomization: order of examples is randomized with a random RNG seed<br>
+     * - Batch size: default (as in the stored DataSets - no splitting/combining)<br>
+     * - File extensions: no filtering - all files in directory are assumed to be a DataSet<br>
+     *
+     * @param rootDir Root directory containing the DataSet objects
+     */
+    public FileDataSetIterator(File rootDir) {
+        this(rootDir, true, new Random(), -1, (String[]) null);
+    }
+
+    /**
+     * Create a FileDataSetIterator with the following default settings:<br>
+     * - Recursive: files in subdirectories are included<br>
+     * - Randomization: order of examples is randomized with a random RNG seed<br>
+     * - Batch size: default (as in the stored DataSets - no splitting/combining)<br>
+     * - File extensions: no filtering - all files in directory are assumed to be a DataSet<br>
+     *
+     * @param rootDirs Root directories containing the DataSet objects. DataSets from all of these directories will
+     *                 be included in the iterator output
+     */
+    public FileDataSetIterator(File... rootDirs) {
+        this(rootDirs, true, new Random(), -1, (String[]) null);
+    }
+
+    /**
+     * Create a FileDataSetIterator with the specified batch size, and the following default settings:<br>
+     * - Recursive: files in subdirectories are included<br>
+     * - Randomization: order of examples is randomized with a random RNG seed<br>
+     * - File extensions: no filtering - all files in directory are assumed to be a DataSet<br>
+     *
+     * @param rootDir   Root directory containing the saved DataSet objects
+     * @param batchSize Batch size. If > 0, DataSets will be split/recombined as required. If <= 0, DataSets will
+     *                  simply be loaded and returned unmodified
+     */
+    public FileDataSetIterator(File rootDir, int batchSize) {
+        this(rootDir, batchSize, (String[]) null);
+    }
+
+    /**
+     * Create a FileDataSetIterator with filtering based on file extensions, and the following default settings:<br>
+     * - Recursive: files in subdirectories are included<br>
+     * - Randomization: order of examples is randomized with a random RNG seed<br>
+     * - Batch size: default (as in the stored DataSets - no splitting/combining)<br>
+     *
+     * @param rootDir         Root directory containing the saved DataSet objects
+     * @param validExtensions May be null. If non-null, only files with one of the specified extensions will be used
+     */
+    public FileDataSetIterator(File rootDir, String... validExtensions) {
+        super(rootDir, -1, validExtensions);
+    }
+
+    /**
+     * Create a FileDataSetIterator with the specified batch size, filtering based on file extensions, and the
+     * following default settings:<br>
+     * - Recursive: files in subdirectories are included<br>
+     * - Randomization: order of examples is randomized with a random RNG seed<br>
+     *
+     * @param rootDir         Root directory containing the saved DataSet objects
+     * @param batchSize       Batch size. If > 0, DataSets will be split/recombined as required. If <= 0, DataSets will
+     *                        simply be loaded and returned unmodified
+     * @param validExtensions May be null. If non-null, only files with one of the specified extensions will be used
+     */
+    public FileDataSetIterator(File rootDir, int batchSize, String... validExtensions) {
+        super(rootDir, batchSize, validExtensions);
+    }
+
+    /**
+     * Create a FileDataSetIterator with all settings specified
+     *
+     * @param rootDir         Root directory containing the saved DataSet objects
+     * @param recursive       If true: include files in subdirectories
+     * @param rng             May be null. If non-null, use this to randomize order
+     * @param batchSize       Batch size. If > 0, DataSets will be split/recombined as required. If <= 0, DataSets will
+     *                        simply be loaded and returned unmodified
+     * @param validExtensions May be null. If non-null, only files with one of the specified extensions will be used
+     */
+    public FileDataSetIterator(File rootDir, boolean recursive, Random rng, int batchSize, String... validExtensions) {
+        this(new File[]{rootDir}, recursive, rng, batchSize, validExtensions);
+    }
+
+    /**
+     * Create a FileDataSetIterator with all settings specified
+     *
+     * @param rootDirs        Root directories containing the DataSet objects. DataSets from all of these directories will
+     *                        be included in the iterator output
+     * @param recursive       If true: include files in subdirectories
+     * @param rng             May be null. If non-null, use this to randomize order
+     * @param batchSize       Batch size. If > 0, DataSets will be split/recombined as required. If <= 0, DataSets will
+     *                        simply be loaded and returned unmodified
+     * @param validExtensions May be null. If non-null, only files with one of the specified extensions will be used
+     */
+    public FileDataSetIterator(File[] rootDirs, boolean recursive, Random rng, int batchSize, String... validExtensions) {
+        super(rootDirs, recursive, rng, batchSize, validExtensions);
+    }
+
+    @Override
+    protected DataSet load(File f) {
+        DataSet ds = new DataSet();
+        ds.load(f);
+        return ds;
+    }
+
+    @Override
+    protected long sizeOf(DataSet of) {
+        return of.numExamples();
+    }
+
+    @Override
+    protected List<DataSet> split(DataSet toSplit) {
+        return toSplit.asList();
+    }
+
+    @Override
+    protected DataSet merge(List<DataSet> toMerge) {
+        return DataSet.merge(toMerge);
+    }
+
+    @Override
+    protected void applyPreprocessor(DataSet toPreProcess) {
+        if (preProcessor != null) {
+            preProcessor.preProcess(toPreProcess);
+        }
+    }
+
+    @Override
+    public DataSet next(int num) {
+        throw new UnsupportedOperationException("Not supported for this iterator");
+    }
+
+    @Override
+    public int inputColumns() {
+        throw new UnsupportedOperationException("Not supported for this iterator");
+    }
+
+    @Override
+    public int totalOutcomes() {
+        throw new UnsupportedOperationException("Not supported for this iterator");
+    }
+
+    @Override
+    public int batch() {
+        return batchSize;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/file/FileMultiDataSetIterator.java

@@ -0,0 +1,170 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.file;
+
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Random;
+
+public class FileMultiDataSetIterator extends BaseFileIterator<MultiDataSet, MultiDataSetPreProcessor> implements MultiDataSetIterator {
+
+
+    /**
+     * Create a FileMultiDataSetIterator with the following default settings:<br>
+     * - Recursive: files in subdirectories are included<br>
+     * - Randomization: order of examples is randomized with a random RNG seed<br>
+     * - Batch size: default (as in the stored DataSets - no splitting/combining)<br>
+     * - File extensions: no filtering - all files in directory are assumed to be a DataSet<br>
+     *
+     * @param rootDir Root directory containing the DataSet objects
+     */
+    public FileMultiDataSetIterator(File rootDir) {
+        this(rootDir, true, new Random(), -1, (String[]) null);
+    }
+
+    /**
+     * Create a FileMultiDataSetIterator with the following default settings:<br>
+     * - Recursive: files in subdirectories are included<br>
+     * - Randomization: order of examples is randomized with a random RNG seed<br>
+     * - Batch size: default (as in the stored DataSets - no splitting/combining)<br>
+     * - File extensions: no filtering - all files in directory are assumed to be a DataSet<br>
+     *
+     * @param rootDirs Root directories containing the MultiDataSet objects. MultiDataSets from all of these
+     *                 directories will be included in the iterator output
+     */
+    public FileMultiDataSetIterator(File... rootDirs) {
+        this(rootDirs, true, new Random(), -1, (String[]) null);
+    }
+
+    /**
+     * Create a FileMultiDataSetIterator with the specified batch size, and the following default settings:<br>
+     * - Recursive: files in subdirectories are included<br>
+     * - Randomization: order of examples is randomized with a random RNG seed<br>
+     * - File extensions: no filtering - all files in directory are assumed to be a DataSet<br>
+     *
+     * @param rootDir   Root directory containing the saved DataSet objects
+     * @param batchSize Batch size. If > 0, DataSets will be split/recombined as required. If <= 0, DataSets will
+     *                  simply be loaded and returned unmodified
+     */
+    public FileMultiDataSetIterator(File rootDir, int batchSize) {
+        this(rootDir, batchSize, (String[]) null);
+    }
+
+    /**
+     * Create a FileMultiDataSetIterator with filtering based on file extensions, and the following default settings:<br>
+     * - Recursive: files in subdirectories are included<br>
+     * - Randomization: order of examples is randomized with a random RNG seed<br>
+     * - Batch size: default (as in the stored DataSets - no splitting/combining)<br>
+     *
+     * @param rootDir         Root directory containing the saved DataSet objects
+     * @param validExtensions May be null. If non-null, only files with one of the specified extensions will be used
+     */
+    public FileMultiDataSetIterator(File rootDir, String... validExtensions) {
+        super(rootDir, -1, validExtensions);
+    }
+
+    /**
+     * Create a FileMultiDataSetIterator with the specified batch size, filtering based on file extensions, and the
+     * following default settings:<br>
+     * - Recursive: files in subdirectories are included<br>
+     * - Randomization: order of examples is randomized with a random RNG seed<br>
+     *
+     * @param rootDir         Root directory containing the saved DataSet objects
+     * @param batchSize       Batch size. If > 0, DataSets will be split/recombined as required. If <= 0, DataSets will
+     *                        simply be loaded and returned unmodified
+     * @param validExtensions May be null. If non-null, only files with one of the specified extensions will be used
+     */
+    public FileMultiDataSetIterator(File rootDir, int batchSize, String... validExtensions) {
+        super(rootDir, batchSize, validExtensions);
+    }
+
+    /**
+     * Create a FileMultiDataSetIterator with all settings specified
+     *
+     * @param rootDir         Root directory containing the saved DataSet objects
+     * @param recursive       If true: include files in subdirectories
+     * @param rng             May be null. If non-null, use this to randomize order
+     * @param batchSize       Batch size. If > 0, DataSets will be split/recombined as required. If <= 0, DataSets will
+     *                        simply be loaded and returned unmodified
+     * @param validExtensions May be null. If non-null, only files with one of the specified extensions will be used
+     */
+    public FileMultiDataSetIterator(File rootDir, boolean recursive, Random rng, int batchSize, String... validExtensions) {
+        this(new File[]{rootDir}, recursive, rng, batchSize, validExtensions);
+    }
+
+    /**
+     * Create a FileMultiDataSetIterator with all settings specified
+     *
+     * @param rootDirs        Root directories containing the MultiDataSet objects. MultiDataSets from all of these
+     *                        directories will be included in the iterator output
+     * @param recursive       If true: include files in subdirectories
+     * @param rng             May be null. If non-null, use this to randomize order
+     * @param batchSize       Batch size. If > 0, DataSets will be split/recombined as required. If <= 0, DataSets will
+     *                        simply be loaded and returned unmodified
+     * @param validExtensions May be null. If non-null, only files with one of the specified extensions will be used
+     */
+    public FileMultiDataSetIterator(File[] rootDirs, boolean recursive, Random rng, int batchSize, String... validExtensions) {
+        super(rootDirs, recursive, rng, batchSize, validExtensions);
+    }
+
+    @Override
+    protected MultiDataSet load(File f) {
+        MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet();
+        try {
+            mds.load(f);
+        } catch (IOException e) {
+            throw new RuntimeException("Error loading MultiDataSet from file: " + f, e);
+        }
+        return mds;
+    }
+
+    @Override
+    protected long sizeOf(MultiDataSet of) {
+        return  of.getFeatures(0).size(0);
+    }
+
+    @Override
+    protected List<MultiDataSet> split(MultiDataSet toSplit) {
+        return toSplit.asList();
+    }
+
+    @Override
+    public MultiDataSet merge(List<MultiDataSet> toMerge) {
+        return org.nd4j.linalg.dataset.MultiDataSet.merge(toMerge);
+    }
+
+    @Override
+    protected void applyPreprocessor(MultiDataSet toPreProcess) {
+        if (preProcessor != null) {
+            preProcessor.preProcess(toPreProcess);
+        }
+    }
+
+    @Override
+    public MultiDataSet next(int num) {
+        throw new UnsupportedOperationException("Not supported for this iterator");
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/loader/DataSetLoaderIterator.java

@@ -0,0 +1,192 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.loader;
+
+import lombok.Data;
+import lombok.Getter;
+import lombok.Setter;
+import org.nd4j.common.loader.Loader;
+import org.nd4j.common.loader.Source;
+import org.nd4j.common.loader.SourceFactory;
+import org.nd4j.common.loader.LocalFileSourceFactory;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.common.util.MathUtils;
+
+import java.io.IOException;
+import java.util.*;
+
+@Data
+public class DataSetLoaderIterator implements DataSetIterator {
+
+    protected final List<String> paths;
+    protected final Iterator<String> iter;
+    protected final SourceFactory sourceFactory;
+    protected final Loader<DataSet> loader;
+    protected final Random rng;
+    protected final int[] order;
+    protected int position;
+
+    @Getter @Setter
+    protected DataSetPreProcessor preProcessor;
+
+    /**
+     * NOTE: When using this constructor (with {@code Iterator<String>}) the DataSetIterator cannot be reset.
+     * Use the other construtor that takes {@code Collection<String>}
+     *
+     * @param paths         Paths to iterate over
+     * @param loader        Loader to use when loading DataSets
+     * @param sourceFactory The factory to use to convert the paths into streams via {@link Source}
+     */
+    public DataSetLoaderIterator(Iterator<String> paths, Loader<DataSet> loader, SourceFactory sourceFactory){
+        this.paths = null;
+        this.iter = paths;
+        this.loader = loader;
+        this.sourceFactory = sourceFactory;
+        this.rng = null;
+        this.order = null;
+    }
+
+    /**
+     * Iterate of the specified collection of strings without randomization
+     *
+     * @param paths         Paths to iterate over
+     * @param loader        Loader to use when loading DataSets
+     * @param sourceFactory The factory to use to convert the paths into streams via {@link Source}
+     */
+    public DataSetLoaderIterator(Collection<String> paths, Loader<DataSet> loader, SourceFactory sourceFactory) {
+        this(paths, null, loader, sourceFactory);
+    }
+
+    /**
+     * Iterate of the specified collection of strings with optional randomization
+     *
+     * @param paths         Paths to iterate over
+     * @param rng           Optional random instance to use for shuffling of order. If null, no shuffling will be used.
+     * @param loader        Loader to use when loading DataSets
+     * @param sourceFactory The factory to use to convert the paths into streams via {@link Source}
+     */
+    public DataSetLoaderIterator(Collection<String> paths, Random rng, Loader<DataSet> loader, SourceFactory sourceFactory){
+        if(paths instanceof List){
+            this.paths = (List<String>)paths;
+        } else {
+            this.paths = new ArrayList<>(paths);
+        }
+        this.rng = rng;
+        this.loader = loader;
+        this.sourceFactory = sourceFactory;
+        this.iter = null;
+
+        if(rng != null){
+            order = new int[paths.size()];
+            for( int i=0; i<order.length; i++ ){
+                order[i] = i;
+            }
+            MathUtils.shuffleArray(order, rng);
+        } else {
+            order = null;
+        }
+    }
+
+    @Override
+    public DataSet next(int i) {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public int inputColumns() {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public int totalOutcomes() {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return paths != null;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return true;
+    }
+
+    @Override
+    public void reset() {
+        if(!resetSupported())
+             throw new UnsupportedOperationException("Reset not supported when using Iterator<String> instead of Iterable<String>");
+        position = 0;
+        if (rng != null) {
+            MathUtils.shuffleArray(order, rng);
+        }
+    }
+
+    @Override
+    public int batch() {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public List<String> getLabels() {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public boolean hasNext() {
+        if(iter != null)
+            return iter.hasNext();
+        return position < paths.size();
+    }
+
+    @Override
+    public DataSet next() {
+        if(!hasNext())
+            throw new NoSuchElementException("No next element");
+        String path;
+        if(iter != null){
+            path = iter.next();
+        } else {
+            if(order != null){
+                path = paths.get(order[position++]);
+            } else {
+                path = paths.get(position++);
+            }
+        }
+        Source s = sourceFactory.getSource(path);
+        DataSet ds;
+        try {
+            ds = loader.load(s);
+        } catch (IOException e){
+            throw new RuntimeException(e);
+        }
+        if(preProcessor != null)
+            preProcessor.preProcess(ds);
+        return ds;
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException("Not supported");
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/loader/MultiDataSetLoaderIterator.java

@@ -0,0 +1,168 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.loader;
+
+import lombok.Data;
+import org.nd4j.common.loader.Loader;
+import org.nd4j.common.loader.Source;
+import org.nd4j.common.loader.SourceFactory;
+import org.nd4j.common.loader.LocalFileSourceFactory;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+import org.nd4j.common.util.MathUtils;
+
+import java.io.IOException;
+import java.util.*;
+
+@Data
+public class MultiDataSetLoaderIterator implements MultiDataSetIterator {
+
+    protected final List<String> paths;
+    protected final Iterator<String> iter;
+    protected final Loader<MultiDataSet> loader;
+    protected final SourceFactory sourceFactory;
+    protected final Random rng;
+    protected final int[] order;
+    protected MultiDataSetPreProcessor preProcessor;
+    protected int position;
+
+    /**
+     * NOTE: When using this constructor (with {@code Iterator<String>}) the MultiDataSetIterator cannot be reset.
+     * Use the other construtor that takes {@code Collection<String>}
+     *
+     * @param paths         Paths to iterate over
+     * @param loader        Loader to use when loading DataSets
+     * @param sourceFactory The factory to use to convert the paths into streams via {@link Source}
+     */
+    public MultiDataSetLoaderIterator(Iterator<String> paths, Loader<MultiDataSet> loader, SourceFactory sourceFactory){
+        this.paths = null;
+        this.iter = paths;
+        this.loader = loader;
+        this.sourceFactory = sourceFactory;
+        this.rng = null;
+        this.order = null;
+    }
+
+    /**
+     * Iterate of the specified collection of strings without randomization
+     *
+     * @param paths         Paths to iterate over
+     * @param loader        Loader to use when loading DataSets
+     * @param sourceFactory The factory to use to convert the paths into streams via {@link Source}
+     */
+    public MultiDataSetLoaderIterator(Collection<String> paths, Loader<MultiDataSet> loader, SourceFactory sourceFactory) {
+        this(paths, null, loader, sourceFactory);
+    }
+
+    /**
+     * Iterate of the specified collection of strings with optional randomization
+     *
+     * @param paths         Paths to iterate over
+     * @param rng           Optional random instance to use for shuffling of order. If null, no shuffling will be used.
+     * @param loader        Loader to use when loading DataSets
+     * @param sourceFactory The factory to use to convert the paths into streams via {@link Source}
+     */
+    public MultiDataSetLoaderIterator(Collection<String> paths, Random rng, Loader<MultiDataSet> loader, SourceFactory sourceFactory) {
+        if(paths instanceof List){
+            this.paths = (List<String>)paths;
+        } else {
+            this.paths = new ArrayList<>(paths);
+        }
+        this.rng = rng;
+        this.loader = loader;
+        this.sourceFactory = sourceFactory;
+        this.iter = null;
+
+        if(rng != null){
+            order = new int[paths.size()];
+            for( int i=0; i<order.length; i++ ){
+                order[i] = i;
+            }
+            MathUtils.shuffleArray(order, rng);
+        } else {
+            order = null;
+        }
+    }
+
+    @Override
+    public MultiDataSet next(int i) {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return paths != null;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return true;
+    }
+
+    @Override
+    public void reset() {
+        if(!resetSupported())
+             throw new UnsupportedOperationException("Reset not supported when using Iterator<String> instead of Iterable<String>");
+        position = 0;
+        if (rng != null) {
+            MathUtils.shuffleArray(order, rng);
+        }
+    }
+
+    @Override
+    public boolean hasNext() {
+        if(iter != null)
+            return iter.hasNext();
+        return position < paths.size();
+    }
+
+    @Override
+    public MultiDataSet next() {
+        if(!hasNext())
+            throw new NoSuchElementException("No next element");
+        String path;
+        if(iter != null){
+            path = iter.next();
+        } else {
+            if(order != null){
+                path = paths.get(order[position++]);
+            } else {
+                path = paths.get(position++);
+            }
+        }
+        Source s = sourceFactory.getSource(path);
+        MultiDataSet mds;
+        try {
+            mds = loader.load(s);
+        } catch (IOException e){
+            throw new RuntimeException(e);
+        }
+        if(preProcessor != null)
+            preProcessor.preProcess(mds);
+        return mds;
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException("Not supported");
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/BaseParallelDataSetIterator.java

@@ -0,0 +1,220 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.parallel;
+
+import lombok.extern.slf4j.Slf4j;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.ParallelDataSetIterator;
+import org.nd4j.linalg.dataset.api.iterator.enums.InequalityHandling;
+import org.nd4j.linalg.exception.ND4JIllegalStateException;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Slf4j
+public abstract class BaseParallelDataSetIterator implements ParallelDataSetIterator {
+    protected AtomicLong counter = new AtomicLong(0);
+
+    protected InequalityHandling inequalityHandling;
+    protected int numProducers;
+
+    protected AtomicBoolean allDepleted = new AtomicBoolean(false);
+    protected MultiBoolean states;
+    protected MultiBoolean resetTracker;
+
+    protected ThreadLocal<Integer> producerAffinity = new ThreadLocal<>();
+
+
+    protected BaseParallelDataSetIterator(int numProducers) {
+        states = new MultiBoolean(numProducers, true);
+        resetTracker = new MultiBoolean(numProducers, false, true);
+        this.numProducers = numProducers;
+    }
+
+
+    public boolean hasNext() {
+        // if all producers are depleted - there's nothing to do here then
+        if (states.allFalse() || allDepleted.get())
+            return false;
+
+        int curIdx = getCurrentProducerIndex();
+
+        boolean hasNext = hasNextFor(curIdx);
+
+        if (hasNext)
+            return true;
+        else
+            states.set(hasNext, curIdx);
+
+        if (states.allFalse())
+            return false;
+
+        switch (inequalityHandling) {
+            // FIXME: RESET should be applicable ONLY to producers which return TRUE for resetSupported();
+            case RESET: {
+                resetTracker.set(true, curIdx);
+
+                // we don't want to have endless loop here, so we only do reset until all producers depleted at least once
+                if (resetTracker.allTrue()) {
+                    allDepleted.set(true);
+                    return false;
+                }
+
+                reset(curIdx);
+
+                // triggering possible adsi underneath
+                hasNextFor(curIdx);
+
+                return true;
+            }
+            case RELOCATE: {
+                // TODO: transparent switch to next producer should happen here
+                while (!hasNext) {
+                    stepForward();
+                    hasNext = hasNextFor(getCurrentProducerIndex());
+                    states.set(hasNext, getCurrentProducerIndex());
+
+                    if (states.allFalse())
+                        return false;
+                }
+
+                return true;
+            }
+            case PASS_NULL: {
+                // we just return true here, no matter what's up
+                return true;
+            }
+            case STOP_EVERYONE: {
+                if (!states.allTrue())
+                    return false;
+
+                return true;
+            }
+            default:
+                throw new ND4JIllegalStateException(
+                        "Unknown InequalityHanding option was passed in: " + inequalityHandling);
+        }
+    }
+
+    public DataSet next() {
+        DataSet ds = nextFor(getCurrentProducerIndex());
+        stepForward();
+        return ds;
+    }
+
+    protected int getCurrentProducerIndex() {
+        return (int) (counter.get() % numProducers);
+    }
+
+    protected void stepForward() {
+        counter.getAndIncrement();
+    }
+
+    @Override
+    public void reset() {
+        for (int i = 0; i < numProducers; i++) {
+            reset(i);
+            states.set(true, i);
+            resetTracker.set(false, i);
+        }
+
+        allDepleted.set(false);
+    }
+
+    @Override
+    public void attachThread(int producer) {
+        producerAffinity.set(producer);
+    }
+
+    @Override
+    public boolean hasNextFor() {
+        if (producerAffinity.get() == null)
+            throw new ND4JIllegalStateException("attachThread(int) should be called prior to this call");
+
+        return hasNextFor(producerAffinity.get());
+    }
+
+    @Override
+    public DataSet nextFor() {
+        if (producerAffinity.get() == null)
+            throw new ND4JIllegalStateException("attachThread(int) should be called prior to this call");
+
+        return nextFor(producerAffinity.get());
+    }
+
+    public abstract boolean hasNextFor(int consumer);
+
+    public abstract DataSet nextFor(int consumer);
+
+    protected abstract void reset(int consumer);
+
+    @Override
+    public int totalOutcomes() {
+        return 0;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return false;
+    }
+
+    @Override
+    public int batch() {
+        return 0;
+    }
+
+    @Override
+    public DataSet next(int num) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int inputColumns() {
+        return 0;
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public DataSetPreProcessor getPreProcessor() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+    @Override
+    public void remove() {
+        // no-op
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/FileSplitParallelDataSetIterator.java

@@ -0,0 +1,130 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.parallel;
+
+import org.nd4j.shade.guava.collect.Lists;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.IOFileFilter;
+import org.apache.commons.io.filefilter.RegexFileFilter;
+import org.nd4j.linalg.dataset.AsyncDataSetIterator;;
+import org.deeplearning4j.datasets.iterator.FileSplitDataSetIterator;
+import org.deeplearning4j.datasets.iterator.callbacks.FileCallback;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.dataset.api.iterator.enums.InequalityHandling;
+import org.nd4j.linalg.exception.ND4JIllegalStateException;
+import org.nd4j.linalg.factory.Nd4j;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+@Slf4j
+public class FileSplitParallelDataSetIterator extends BaseParallelDataSetIterator {
+
+    public static final String DEFAULT_PATTERN = "dataset-%d.bin";
+    private String pattern;
+    private int buffer;
+
+    protected List<DataSetIterator> asyncIterators = new ArrayList<>();
+
+    public FileSplitParallelDataSetIterator(@NonNull File rootFolder, @NonNull String pattern,
+                    @NonNull FileCallback callback) {
+        this(rootFolder, pattern, callback, Nd4j.getAffinityManager().getNumberOfDevices());
+    }
+
+    public FileSplitParallelDataSetIterator(@NonNull File rootFolder, @NonNull String pattern,
+                    @NonNull FileCallback callback, int numThreads) {
+        this(rootFolder, pattern, callback, numThreads, InequalityHandling.STOP_EVERYONE);
+    }
+
+    public FileSplitParallelDataSetIterator(@NonNull File rootFolder, @NonNull String pattern,
+                    @NonNull FileCallback callback, int numThreads, @NonNull InequalityHandling inequalityHandling) {
+        this(rootFolder, pattern, callback, numThreads, 2, inequalityHandling);
+    }
+
+    public FileSplitParallelDataSetIterator(@NonNull File rootFolder, @NonNull String pattern,
+                    @NonNull FileCallback callback, int numThreads, int bufferPerThread,
+                    @NonNull InequalityHandling inequalityHandling) {
+        super(numThreads);
+
+        if (!rootFolder.exists() || !rootFolder.isDirectory())
+            throw new IllegalArgumentException("Root folder should point to existing folder");
+
+        this.pattern = pattern;
+        this.inequalityHandling = inequalityHandling;
+        this.buffer = bufferPerThread;
+
+        String modifiedPattern = pattern.replaceAll("\\%d", ".*.");
+
+        IOFileFilter fileFilter = new RegexFileFilter(modifiedPattern);
+
+
+        List<File> files = new ArrayList<>(FileUtils.listFiles(rootFolder, fileFilter, null));
+        log.debug("Files found: {}; Producers: {}", files.size(), numProducers);
+
+        if (files.isEmpty())
+            throw new IllegalArgumentException("No suitable files were found");
+
+        int numDevices = Nd4j.getAffinityManager().getNumberOfDevices();
+        int cnt = 0;
+        for (List<File> part : Lists.partition(files, files.size() / numThreads)) {
+            // discard remainder
+            if (cnt >= numThreads)
+                break;
+
+            int cDev = cnt % numDevices;
+            asyncIterators.add(new AsyncDataSetIterator(new FileSplitDataSetIterator(part, callback), bufferPerThread,
+                            true, cDev));
+            cnt++;
+        }
+
+    }
+
+    @Override
+    public boolean hasNextFor(int consumer) {
+        if (consumer >= numProducers || consumer < 0)
+            throw new ND4JIllegalStateException("Non-existent consumer was requested");
+
+        return asyncIterators.get(consumer).hasNext();
+    }
+
+    @Override
+    public DataSet nextFor(int consumer) {
+        if (consumer >= numProducers || consumer < 0)
+            throw new ND4JIllegalStateException("Non-existent consumer was requested");
+
+        return asyncIterators.get(consumer).next();
+    }
+
+    @Override
+    protected void reset(int consumer) {
+        if (consumer >= numProducers || consumer < 0)
+            throw new ND4JIllegalStateException("Non-existent consumer was requested");
+
+        asyncIterators.get(consumer).reset();
+    }
+
+
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/JointParallelDataSetIterator.java

@@ -0,0 +1,155 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.parallel;
+
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.nd4j.linalg.dataset.AsyncDataSetIterator;;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.dataset.api.iterator.enums.InequalityHandling;
+import org.nd4j.linalg.exception.ND4JIllegalStateException;
+import org.nd4j.linalg.factory.Nd4j;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Slf4j
+public class JointParallelDataSetIterator extends BaseParallelDataSetIterator {
+    protected List<DataSetIterator> asyncIterators = new ArrayList<>();
+    protected boolean enforceSingleDevice;
+    protected int bufferSizePerDevice;
+
+
+    public JointParallelDataSetIterator(@NonNull List<DataSetIterator> iterators, boolean singleDeviceMode,
+                    int bufferSize, @NonNull InequalityHandling inequalityHandling) {
+        super(iterators.size());
+        this.enforceSingleDevice = singleDeviceMode;
+        this.bufferSizePerDevice = bufferSize;
+        this.numProducers = iterators.size();
+        this.inequalityHandling = inequalityHandling;
+
+        if (numProducers == 0)
+            throw new IllegalArgumentException("You can't start ParallelDataSetIterator without input data");
+
+        initializeIterators(iterators);
+    }
+
+    protected void initializeIterators(List<DataSetIterator> originals) {
+        int numDevices = Nd4j.getAffinityManager().getNumberOfDevices();
+
+        int currentDevice = Nd4j.getAffinityManager().getDeviceForCurrentThread();
+
+        if (originals.size() % numDevices != 0)
+            log.error("WARNING: number of splits doesn't match number of devices!");
+
+        int cnt = 0;
+        for (DataSetIterator iterator : originals) {
+            int cDev = cnt % numDevices;
+            asyncIterators.add(new AsyncDataSetIterator(iterator, bufferSizePerDevice, true, cDev));
+            cnt++;
+        }
+    }
+
+    public boolean hasNextFor(int consumer) {
+        if (consumer >= numProducers || consumer < 0)
+            throw new ND4JIllegalStateException("Non-existent consumer was requested");
+
+        return asyncIterators.get(consumer).hasNext();
+    }
+
+
+    public DataSet nextFor(int consumer) {
+        if (consumer >= numProducers || consumer < 0)
+            throw new ND4JIllegalStateException("Non-existent consumer was requested");
+
+        return asyncIterators.get(consumer).next();
+    }
+
+    protected void reset(int consumer) {
+        if (consumer >= numProducers || consumer < 0)
+            throw new ND4JIllegalStateException("Non-existent consumer was requested");
+
+        asyncIterators.get(consumer).reset();
+    }
+
+
+    public static class Builder {
+        private List<DataSetIterator> iterators = new ArrayList<>();
+        private boolean enforceSingleDevice = true;
+        private int bufferSize = 4;
+        private InequalityHandling inequalityHandling;
+
+        public Builder(@NonNull InequalityHandling inequalityHandling) {
+            this.inequalityHandling = inequalityHandling;
+        }
+
+        public Builder(@NonNull List<DataSetIterator> iterators, @NonNull InequalityHandling inequalityHandling) {
+            this.inequalityHandling = inequalityHandling;
+
+            for (DataSetIterator iterator : iterators)
+                addSourceIterator(iterator);
+        }
+
+
+        public Builder addSourceIterator(@NonNull DataSetIterator iterator) {
+            if (!iterator.asyncSupported())
+                throw new IllegalArgumentException("Source iterators should support async mode");
+
+            //TODO: add strict equality check here, we don't want it equal
+            if (!hasIterator(iterator))
+                iterators.add(iterator);
+            else
+                throw new IllegalArgumentException("You can't put equal iterators into this joint iterator");
+
+            return this;
+        }
+
+        protected boolean hasIterator(DataSetIterator iterator) {
+            for (DataSetIterator iter : iterators) {
+                if (iter == iterator)
+                    return true;
+            }
+
+            return false;
+        }
+
+        public Builder setBufferSizePerSplit(int bufferSize) {
+            this.bufferSize = bufferSize;
+            return this;
+        }
+
+
+        public Builder enforceSingleDevice(boolean reallyEnforce) {
+            this.enforceSingleDevice = reallyEnforce;
+            return this;
+        }
+
+
+        public JointParallelDataSetIterator build() {
+            JointParallelDataSetIterator jpdsi = new JointParallelDataSetIterator(iterators, enforceSingleDevice,
+                            bufferSize, inequalityHandling);
+
+            return jpdsi;
+        }
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/MultiBoolean.java

@@ -0,0 +1,113 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.parallel;
+
+import lombok.extern.slf4j.Slf4j;
+import org.nd4j.linalg.exception.ND4JIllegalStateException;
+
+@Slf4j
+public class MultiBoolean {
+    private final int numEntries;
+    private int holder = 0;
+    private int max = 0;
+    private boolean oneTime;
+    private MultiBoolean timeTracker;
+
+    public MultiBoolean(int numEntries) {
+        this(numEntries, false);
+    }
+
+    public MultiBoolean(int numEntries, boolean initialValue) {
+        this(numEntries, initialValue, false);
+    }
+
+    public MultiBoolean(int numEntries, boolean initialValue, boolean oneTime) {
+        if (numEntries > 32)
+            throw new UnsupportedOperationException("Up to 32 entries can be tracked at once.");
+
+        this.oneTime = oneTime;
+        this.numEntries = numEntries;
+        for (int i = 1; i <= numEntries; i++) {
+            this.max |= 1 << i;
+        }
+
+        if (initialValue)
+            this.holder = this.max;
+
+        if (oneTime)
+            this.timeTracker = new MultiBoolean(numEntries, false, false);
+    }
+
+    /**
+     * Sets specified entry to specified state
+     *
+     * @param value
+     * @param entry
+     */
+    public void set(boolean value, int entry) {
+        if (entry > numEntries || entry < 0)
+            throw new ND4JIllegalStateException(
+                            "Entry index given (" + entry + ")in is higher then configured one (" + numEntries + ")");
+
+        if (oneTime && this.timeTracker.get(entry))
+            return;
+
+        if (value)
+            this.holder |= 1 << (entry + 1);
+        else
+            this.holder &= ~(1 << (entry + 1));
+
+        if (oneTime)
+            this.timeTracker.set(true, entry);
+    }
+
+    /**
+     * Gets current state for specified entry
+     *
+     * @param entry
+     * @return
+     */
+    public boolean get(int entry) {
+        if (entry > numEntries || entry < 0)
+            throw new ND4JIllegalStateException(
+                            "Entry index given (" + entry + ")in is higher then configured one (" + numEntries + ")");
+
+        return (this.holder & 1 << (entry + 1)) != 0;
+    }
+
+    /**
+     * This method returns true if ALL states are true. False otherwise.
+     *
+     * @return
+     */
+    public boolean allTrue() {
+        //log.info("Holder: {}; Max: {}", holder, max);
+        return holder == max;
+    }
+
+    /**
+     * This method returns true if ALL states are false. False otherwise
+     * @return
+     */
+    public boolean allFalse() {
+        return holder == 0;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/utilty/BenchmarkDataSetIterator.java

@@ -0,0 +1,184 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.utilty;
+
+import lombok.extern.slf4j.Slf4j;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.indexing.NDArrayIndex;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Slf4j
+public class BenchmarkDataSetIterator implements DataSetIterator {
+    private INDArray baseFeatures;
+    private INDArray baseLabels;
+    private long limit;
+    private AtomicLong counter = new AtomicLong(0);
+
+    /**
+     * @param featuresShape   Shape of the features data to randomly generate
+     * @param numLabels       Number of label classes (for classification)
+     * @param totalIterations Total number of iterations per epoch
+     */
+    public BenchmarkDataSetIterator(int[] featuresShape, int numLabels, int totalIterations) {
+        this(featuresShape, numLabels, totalIterations, -1, -1);
+    }
+
+    /**
+     * Creates 2d (shape [minibatch, numLabels]) or 4d labels ([minibatch, numLabels, gridWidth, gridHeight]),
+     * depending on value of gridWidth and gridHeight.
+     *
+     * @param featuresShape   Shape of the features data to randomly generate
+     * @param numLabels       Number of label classes (for classification)
+     * @param totalIterations Total number of iterations
+     * @param gridWidth       If > 0, use to create 4d labels
+     * @param gridHeight      If > 0, use to create 4d labels
+     */
+    public BenchmarkDataSetIterator(int[] featuresShape, int numLabels, int totalIterations, int gridWidth, int gridHeight) {
+        this.baseFeatures = Nd4j.rand(featuresShape);
+        this.baseLabels = gridWidth > 0 && gridHeight > 0
+                        ? Nd4j.create(featuresShape[0], numLabels, gridWidth, gridHeight)
+                        : Nd4j.create(featuresShape[0], numLabels);
+        if(this.baseLabels.rank() == 2){
+            this.baseLabels.getColumn(1).assign(1.0);
+        } else {
+            this.baseLabels.get(NDArrayIndex.all(), NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.all());
+        }
+
+        Nd4j.getExecutioner().commit();
+        this.limit = totalIterations;
+    }
+
+    /**
+     * @param example         DataSet to return on each call of next()
+     * @param totalIterations Total number of iterations
+     */
+    public BenchmarkDataSetIterator(DataSet example, int totalIterations) {
+        this.baseFeatures = example.getFeatures().dup();
+        this.baseLabels = example.getLabels().dup();
+
+        Nd4j.getExecutioner().commit();
+        this.limit = totalIterations;
+    }
+
+    @Override
+    public DataSet next(int i) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int inputColumns() {
+        return 0;
+    }
+
+    @Override
+    public int totalOutcomes() {
+        return 0;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return true;
+    }
+
+    @Override
+    public void reset() {
+        this.counter.set(0);
+    }
+
+    @Override
+    public int batch() {
+        return 0;
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor dataSetPreProcessor) {
+
+    }
+
+    @Override
+    public DataSetPreProcessor getPreProcessor() {
+        return null;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+    /**
+     * Returns {@code true} if the iteration has more elements.
+     * (In other words, returns {@code true} if {@link #next} would
+     * return an element rather than throwing an exception.)
+     *
+     * @return {@code true} if the iteration has more elements
+     */
+    @Override
+    public boolean hasNext() {
+        return counter.get() < limit;
+    }
+
+    /**
+     * Returns the next element in the iteration.
+     *
+     * @return the next element in the iteration
+     */
+    @Override
+    public DataSet next() {
+        counter.incrementAndGet();
+
+        DataSet ds = new DataSet(baseFeatures, baseLabels);
+
+        return ds;
+    }
+
+    /**
+     * Removes from the underlying collection the last element returned
+     * by this iterator (optional operation).  This method can be called
+     * only once per call to {@link #next}.  The behavior of an iterator
+     * is unspecified if the underlying collection is modified while the
+     * iteration is in progress in any way other than by calling this
+     * method.
+     *
+     * @throws UnsupportedOperationException if the {@code remove}
+     *                                       operation is not supported by this iterator
+     * @throws IllegalStateException         if the {@code next} method has not
+     *                                       yet been called, or the {@code remove} method has already
+     *                                       been called after the last call to the {@code next}
+     *                                       method
+     * @implSpec The default implementation throws an instance of
+     * {@link UnsupportedOperationException} and performs no other action.
+     */
+    @Override
+    public void remove() {
+
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/utilty/BenchmarkMultiDataSetIterator.java

@@ -0,0 +1,153 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.utilty;
+
+import lombok.extern.slf4j.Slf4j;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.MultiDataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+import org.nd4j.linalg.factory.Nd4j;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+@Slf4j
+public class BenchmarkMultiDataSetIterator implements MultiDataSetIterator {
+    private INDArray[] baseFeatures;
+    private INDArray[] baseLabels;
+    private long limit;
+    private AtomicLong counter = new AtomicLong(0);
+
+    public BenchmarkMultiDataSetIterator(int[][] featuresShape, int[] numLabels, int totalIterations) {
+        if (featuresShape.length != numLabels.length)
+            throw new IllegalArgumentException("Number of input features must match length of input labels.");
+
+        this.baseFeatures = new INDArray[featuresShape.length];
+        for (int i = 0; i < featuresShape.length; i++) {
+            baseFeatures[i] = Nd4j.rand(featuresShape[i]);
+        }
+        this.baseLabels = new INDArray[featuresShape.length];
+        for (int i = 0; i < featuresShape.length; i++) {
+            baseLabels[i] = Nd4j.create(featuresShape[i][0], numLabels[i]);
+            baseLabels[i].getColumn(1).assign(1.0);
+        }
+
+        Nd4j.getExecutioner().commit();
+        this.limit = totalIterations;
+    }
+
+    public BenchmarkMultiDataSetIterator(MultiDataSet example, int totalIterations) {
+        this.baseFeatures = new INDArray[example.getFeatures().length];
+        for (int i = 0; i < example.getFeatures().length; i++) {
+            baseFeatures[i] = example.getFeatures()[i].dup();
+        }
+        this.baseLabels = new INDArray[example.getLabels().length];
+        for (int i = 0; i < example.getLabels().length; i++) {
+            baseFeatures[i] = example.getLabels()[i].dup();
+        }
+
+        Nd4j.getExecutioner().commit();
+        this.limit = totalIterations;
+    }
+
+    @Override
+    public MultiDataSet next(int i) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return true;
+    }
+
+    @Override
+    public void reset() {
+        this.counter.set(0);
+    }
+
+    @Override
+    public void setPreProcessor(MultiDataSetPreProcessor dataSetPreProcessor) {
+
+    }
+
+    @Override
+    public MultiDataSetPreProcessor getPreProcessor() {
+        return null;
+    }
+
+    /**
+     * Returns {@code true} if the iteration has more elements.
+     * (In other words, returns {@code true} if {@link #next} would
+     * return an element rather than throwing an exception.)
+     *
+     * @return {@code true} if the iteration has more elements
+     */
+    @Override
+    public boolean hasNext() {
+        return counter.get() < limit;
+    }
+
+    /**
+     * Returns the next element in the iteration.
+     *
+     * @return the next element in the iteration
+     */
+    @Override
+    public MultiDataSet next() {
+        counter.incrementAndGet();
+
+        INDArray[] features = new INDArray[baseFeatures.length];
+        System.arraycopy(baseFeatures, 0, features, 0, baseFeatures.length);
+        INDArray[] labels = new INDArray[baseLabels.length];
+        System.arraycopy(baseLabels, 0, labels, 0, baseLabels.length);
+
+        MultiDataSet ds = new MultiDataSet(features, labels);
+
+        return ds;
+    }
+
+    /**
+     * Removes from the underlying collection the last element returned
+     * by this iterator (optional operation).  This method can be called
+     * only once per call to {@link #next}.  The behavior of an iterator
+     * is unspecified if the underlying collection is modified while the
+     * iteration is in progress in any way other than by calling this
+     * method.
+     *
+     * @throws UnsupportedOperationException if the {@code remove}
+     *                                       operation is not supported by this iterator
+     * @throws IllegalStateException         if the {@code next} method has not
+     *                                       yet been called, or the {@code remove} method has already
+     *                                       been called after the last call to the {@code next}
+     *                                       method
+     * @implSpec The default implementation throws an instance of
+     * {@link UnsupportedOperationException} and performs no other action.
+     */
+    @Override
+    public void remove() {
+
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/utilty/ListDataSetIterator.java

@@ -0,0 +1,140 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.utilty;
+
+import lombok.Getter;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class ListDataSetIterator<T extends DataSet> implements DataSetIterator {
+
+    private static final long serialVersionUID = -7569201667767185411L;
+    private int curr = 0;
+    private int batch = 10;
+    private List<T> list;
+    @Getter
+    private DataSetPreProcessor preProcessor;
+
+    /**
+     * @param coll  Collection of datasets with 1 example each
+     * @param batch Batch size
+     */
+    public ListDataSetIterator(Collection<T> coll, int batch) {
+        list = new ArrayList<>(coll);
+        this.batch = batch;
+
+    }
+
+    /**
+     * Initializes with a batch of 5
+     *
+     * @param coll the collection to iterate over
+     */
+    public ListDataSetIterator(Collection<T> coll) {
+        this(coll, 5);
+
+    }
+
+    @Override
+    public synchronized boolean hasNext() {
+        return curr < list.size();
+    }
+
+    @Override
+    public synchronized DataSet next() {
+        return next(batch);
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int inputColumns() {
+        return list.get(0).getFeatures().columns();
+    }
+
+    @Override
+    public int totalOutcomes() {
+        return list.get(0).getLabels().columns();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        //Already in memory -> doesn't make sense to prefetch
+        return false;
+    }
+
+    @Override
+    public synchronized void reset() {
+        curr = 0;
+    }
+
+    @Override
+    public int batch() {
+        return batch;
+    }
+
+    @Override
+    public void setPreProcessor(DataSetPreProcessor preProcessor) {
+        this.preProcessor = preProcessor;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+
+    @Override
+    public DataSet next(int num) {
+        int end = curr + num;
+
+        List<DataSet> r = new ArrayList<>();
+        if (end >= list.size())
+            end = list.size();
+        for (; curr < end; curr++) {
+            r.add(list.get(curr));
+        }
+
+        DataSet d = DataSet.merge(r);
+        if (preProcessor != null) {
+            if (!d.isPreProcessed()) {
+                preProcessor.preProcess(d);
+                d.markAsPreProcessed();
+            }
+        }
+        return d;
+    }
+
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/utilty/SingletonDataSetIterator.java

@@ -0,0 +1,109 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.utilty;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public class SingletonDataSetIterator implements DataSetIterator {
+
+    private final DataSet dataSet;
+    private boolean hasNext = true;
+    private boolean preprocessed = false;
+    @Getter @Setter
+    private DataSetPreProcessor preProcessor;
+
+    /**
+     * @param multiDataSet The underlying dataset to return
+     */
+    public SingletonDataSetIterator(DataSet multiDataSet) {
+        this.dataSet = multiDataSet;
+    }
+
+    @Override
+    public DataSet next(int num) {
+        return next();
+    }
+
+    @Override
+    public int inputColumns() {
+        return 0;
+    }
+
+    @Override
+    public int totalOutcomes() {
+        return 0;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return false;
+    }
+
+    @Override
+    public void reset() {
+        hasNext = true;
+    }
+
+    @Override
+    public int batch() {
+        return 0;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+    @Override
+    public boolean hasNext() {
+        return hasNext;
+    }
+
+    @Override
+    public DataSet next() {
+        if (!hasNext) {
+            throw new NoSuchElementException("No elements remaining");
+        }
+        hasNext = false;
+        if (preProcessor != null && !preprocessed) {
+            preProcessor.preProcess(dataSet);
+            preprocessed = true;
+        }
+        return dataSet;
+    }
+
+    @Override
+    public void remove() {
+        //No op
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/utilty/SingletonMultiDataSetIterator.java

@@ -0,0 +1,95 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.utilty;
+
+import org.nd4j.linalg.dataset.api.MultiDataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
+
+import java.util.NoSuchElementException;
+
+public class SingletonMultiDataSetIterator implements MultiDataSetIterator {
+
+    private final MultiDataSet multiDataSet;
+    private boolean hasNext = true;
+    private boolean preprocessed = false;
+    private MultiDataSetPreProcessor preProcessor;
+
+    /**
+     * @param multiDataSet The underlying MultiDataSet to return
+     */
+    public SingletonMultiDataSetIterator(MultiDataSet multiDataSet) {
+        this.multiDataSet = multiDataSet;
+    }
+
+    @Override
+    public MultiDataSet next(int num) {
+        return next();
+    }
+
+    @Override
+    public void setPreProcessor(MultiDataSetPreProcessor preProcessor) {
+        this.preProcessor = preProcessor;
+    }
+
+    @Override
+    public MultiDataSetPreProcessor getPreProcessor() {
+        return preProcessor;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public boolean asyncSupported() {
+        return false;
+    }
+
+    @Override
+    public void reset() {
+        hasNext = true;
+    }
+
+    @Override
+    public boolean hasNext() {
+        return hasNext;
+    }
+
+    @Override
+    public MultiDataSet next() {
+        if (!hasNext) {
+            throw new NoSuchElementException("No elements remaining");
+        }
+        hasNext = false;
+        if (preProcessor != null && !preprocessed) {
+            preProcessor.preProcess(multiDataSet);
+            preprocessed = true;
+        }
+        return multiDataSet;
+    }
+
+    @Override
+    public void remove() {
+        //No op
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java9/module-info.java

@@ -0,0 +1,16 @@
+module deeplearning4j.utility.iterators {
+    requires commons.io;
+    requires guava;
+
+    requires transitive nd4j.api;
+    requires transitive nd4j.common;
+    requires transitive slf4j.api;
+
+    exports org.deeplearning4j.datasets.iterator;
+    exports org.deeplearning4j.datasets.iterator.callbacks;
+    exports org.deeplearning4j.datasets.iterator.file;
+    exports org.deeplearning4j.datasets.iterator.loader;
+    exports org.deeplearning4j.datasets.iterator.parallel;
+    exports org.deeplearning4j.datasets.iterator.utilty;
+
+}

deeplearning4j/deeplearning4j-data/pom.xml

@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ /* ******************************************************************************
+  ~  *
+  ~  *
+  ~  * This program and the accompanying materials are made available under the
+  ~  * terms of the Apache License, Version 2.0 which is available at
+  ~  * https://www.apache.org/licenses/LICENSE-2.0.
+  ~  *
+  ~  *  See the NOTICE file distributed with this work for additional
+  ~  *  information regarding copyright ownership.
+  ~  * Unless required by applicable law or agreed to in writing, software
+  ~  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  ~  * License for the specific language governing permissions and limitations
+  ~  * under the License.
+  ~  *
+  ~  * SPDX-License-Identifier: Apache-2.0
+  ~  ******************************************************************************/
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>deeplearning4j-parent</artifactId>
+        <groupId>org.eclipse.deeplearning4j</groupId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>deeplearning4j-data</artifactId>
+    <packaging>pom</packaging>
+
+    <name>deeplearning4j-data</name>
+
+    <modules>
+        <module>deeplearning4j-datavec-iterators</module>
+        <module>deeplearning4j-datasets</module>
+        <module>deeplearning4j-utility-iterators</module>
+    </modules>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.eclipse.deeplearning4j</groupId>
+                <artifactId>nd4j-api</artifactId>
+                <version>${nd4j.version}</version>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+
+
+
+</project>