chore: import upstream snapshot with attribution

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/pom.xml

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ /* ******************************************************************************
+  ~  *
+  ~  *
+  ~  * This program and the accompanying materials are made available under the
+  ~  * terms of the Apache License, Version 2.0 which is available at
+  ~  * https://www.apache.org/licenses/LICENSE-2.0.
+  ~  *
+  ~  *  See the NOTICE file distributed with this work for additional
+  ~  *  information regarding copyright ownership.
+  ~  * Unless required by applicable law or agreed to in writing, software
+  ~  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  ~  * License for the specific language governing permissions and limitations
+  ~  * under the License.
+  ~  *
+  ~  * SPDX-License-Identifier: Apache-2.0
+  ~  ******************************************************************************/
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.eclipse.deeplearning4j</groupId>
+        <artifactId>deeplearning4j-data</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>deeplearning4j-datasets</artifactId>
+    <packaging>jar</packaging>
+
+    <name>deeplearning4j-datasets</name>
+
+    <properties>
+        <module.name>deeplearning4j.datasets</module.name>
+    </properties>
+
+    <build>
+        <plugins>
+           <plugin>
+                <groupId>org.moditect</groupId>
+                <artifactId>moditect-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>datavec-data-image</artifactId>
+            <version>${datavec.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>deeplearning4j-datavec-iterators</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>resources</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+
+
+</project>

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/base/EmnistFetcher.java

@@ -0,0 +1,127 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.base;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.eclipse.deeplearning4j.resources.DataSetResource;
+import org.eclipse.deeplearning4j.resources.ResourceDataSets;
+import org.eclipse.deeplearning4j.resources.utils.EMnistSet;
+
+import java.io.File;
+import java.io.IOException;
+
+@Slf4j
+public class EmnistFetcher extends MnistFetcher {
+
+    private final EMnistSet ds;
+    @Getter
+    private DataSetResource emnistDataTrain;
+    @Getter
+    private DataSetResource emnistDataTest;
+    @Getter
+    private DataSetResource emnistLabelsTrain;
+    @Getter
+    private DataSetResource emnistLabelsTest;
+    @Getter
+    private DataSetResource emnistMappingTrain;
+    @Getter
+    private DataSetResource emnistMappingTest;
+
+    public EmnistFetcher() {
+        this(EMnistSet.MNIST);
+    }
+
+    public EmnistFetcher(EMnistSet ds) {
+        this.ds = ds;
+        emnistDataTrain = ResourceDataSets.emnistTrain(ds);
+        emnistDataTest = ResourceDataSets.emnistTest(ds);
+        emnistLabelsTrain = ResourceDataSets.emnistLabelsTrain(ds);
+        emnistLabelsTest = ResourceDataSets.emnistLabelsTest(ds);
+        emnistMappingTrain = ResourceDataSets.emnistMappingTrain(ds);
+        emnistMappingTest = ResourceDataSets.emnistMappingTest(ds);
+
+    }
+
+    public EmnistFetcher(EMnistSet ds,File topLevelDir) {
+        this.ds = ds;
+        emnistDataTrain = ResourceDataSets.emnistTrain(ds,topLevelDir);
+        emnistDataTest = ResourceDataSets.emnistTest(ds,topLevelDir);
+        emnistLabelsTrain = ResourceDataSets.emnistLabelsTrain(ds,topLevelDir);
+        emnistLabelsTest = ResourceDataSets.emnistLabelsTest(ds,topLevelDir);
+        emnistMappingTrain = ResourceDataSets.emnistMappingTrain(ds,topLevelDir);
+        emnistMappingTest = ResourceDataSets.emnistMappingTest(ds,topLevelDir);
+
+    }
+
+
+    @Override
+    public String getName() {
+        return "EMNIST";
+    }
+
+    // --- Train files ---
+
+    public static int numLabels(EMnistSet dataSet) {
+        switch (dataSet) {
+            case COMPLETE:
+                return 62;
+            case MERGE:
+                return 47;
+            case BALANCED:
+                return 47;
+            case LETTERS:
+                return 26;
+            case DIGITS:
+                return 10;
+            case MNIST:
+                return 10;
+            default:
+                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
+        }
+    }
+
+    @Override
+    public File downloadAndUntar() throws IOException {
+        if (fileDir != null) {
+            return fileDir;
+        }
+
+        File baseDir = getBaseDir();
+        if (!(baseDir.isDirectory() || baseDir.mkdir())) {
+            throw new IOException("Could not mkdir " + baseDir);
+        }
+
+        log.info("Downloading {}...", getName());
+        // get features
+        emnistDataTrain.download(true,3,300000,30000);
+        emnistDataTest.download(true,3,300000,30000);
+        emnistLabelsTrain.download(true,3,300000,30000);
+        emnistLabelsTest.download(true,3,300000,30000);
+        emnistMappingTrain.download(false,3,300000,30000);
+        emnistMappingTest.download(false,3,300000,30000);
+
+
+        // get labels
+        fileDir = baseDir;
+        return fileDir;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/base/IrisUtils.java

@@ -0,0 +1,90 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.base;
+
+import org.apache.commons.io.IOUtils;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.common.resources.Downloader;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+public class IrisUtils {
+
+    private static final String IRIS_RELATIVE_URL = "datasets/iris.dat";
+    private static final String MD5 = "1c21400a78061197eac64c6748844216";
+
+    private IrisUtils() {}
+
+    public static List<DataSet> loadIris(int from, int to) throws IOException {
+        File rootDir = DL4JResources.getDirectory(ResourceType.DATASET, "iris");
+        File irisData = new File(rootDir, "iris.dat");
+        if(!irisData.exists()) {
+            URL url = DL4JResources.getURL(IRIS_RELATIVE_URL);
+            Downloader.download("Iris", url, irisData, MD5, 3);
+        }
+
+        @SuppressWarnings("unchecked")
+        List<String> lines;
+        try(InputStream is = new FileInputStream(irisData)){
+            lines = IOUtils.readLines(is);
+        }
+        List<DataSet> list = new ArrayList<>();
+        INDArray ret = to - from > 1 ? Nd4j.ones(Math.abs(to - from), 4) : Nd4j.ones( 4);
+        double[][] outcomes = new double[lines.size()][3];
+        int putCount = 0;
+
+        for (int i = from; i < to; i++) {
+            String line = lines.get(i);
+            String[] split = line.split(",");
+
+            addRow(ret, putCount++, split);
+
+            String outcome = split[split.length - 1];
+            double[] rowOutcome = new double[3];
+            rowOutcome[Integer.parseInt(outcome)] = 1;
+            outcomes[i] = rowOutcome;
+        }
+
+        for (int i = 0; i < ret.rows(); i++) {
+            DataSet add = new DataSet(ret.getRow(i, false), Nd4j.create(outcomes[from + i], 3));
+            list.add(add);
+        }
+        return list;
+    }
+
+    private static void addRow(INDArray ret, int row, String[] line) {
+        double[] vector = new double[4];
+        for (int i = 0; i < 4; i++)
+            vector[i] = Double.parseDouble(line[i]);
+
+        ret.putRow(row, Nd4j.create(vector));
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/base/MnistFetcher.java

@@ -0,0 +1,89 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.base;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.eclipse.deeplearning4j.resources.DataSetResource;
+import org.eclipse.deeplearning4j.resources.ResourceDataSets;
+import org.nd4j.common.resources.Downloader;
+
+import java.io.*;
+import java.net.URL;
+
+@Data
+@NoArgsConstructor
+@Slf4j
+public class MnistFetcher {
+
+    protected static final String LOCAL_DIR_NAME = "MNIST";
+
+    protected File fileDir;
+
+    private DataSetResource mnistTrain = ResourceDataSets.mnistTrain();
+    private DataSetResource mnistTest = ResourceDataSets.mnistTest();
+    private DataSetResource mnistTrainLabels = ResourceDataSets.mnistTrainLabels();
+    private DataSetResource mnistTestLabels = ResourceDataSets.mnistTestLabels();
+
+
+    public MnistFetcher(File tempDir) {
+        this.fileDir = tempDir;
+    }
+
+
+    public String getName() {
+        return "MNIST";
+    }
+
+    public File getBaseDir() {
+        return DL4JResources.getDirectory(ResourceType.DATASET, getName());
+    }
+
+
+
+    public File downloadAndUntar() throws IOException {
+        if (fileDir != null) {
+            return fileDir;
+        }
+
+        File baseDir = getBaseDir();
+        if (!(baseDir.isDirectory() || baseDir.mkdir())) {
+            throw new IOException("Could not mkdir " + baseDir);
+        }
+
+        log.info("Downloading {}...", getName());
+        // get features
+
+
+        mnistTrain.download(true,3,200000,20000);
+        mnistTest.download(true,3,200000,20000);
+
+        mnistTrainLabels.download(true,3,200000,20000);
+        mnistTestLabels.download(true,3,200000,20000);
+
+        // get labels
+        fileDir = baseDir;
+        return fileDir;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/CacheableDataSet.java

@@ -0,0 +1,37 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.image.transform.ImageTransform;
+
+interface CacheableDataSet {
+
+    String remoteDataUrl();
+    String remoteDataUrl(DataSetType set);
+    String localCacheName();
+    String dataSetName(DataSetType set);
+    long expectedChecksum();
+    long expectedChecksum(DataSetType set);
+    boolean isCached();
+    RecordReader getRecordReader(long rngSeed, int[] imgDim, DataSetType set, ImageTransform imageTransform);
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/CacheableExtractableDataSetFetcher.java

@@ -0,0 +1,125 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.nd4j.common.util.ArchiveUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.zip.Adler32;
+import java.util.zip.Checksum;
+
+@Slf4j
+public abstract class CacheableExtractableDataSetFetcher implements CacheableDataSet {
+
+    @Override public String dataSetName(DataSetType set) { return ""; }
+    @Override public String remoteDataUrl() { return remoteDataUrl(DataSetType.TRAIN); }
+    @Override public long expectedChecksum() { return expectedChecksum(DataSetType.TRAIN); }
+    public void downloadAndExtract() throws IOException { downloadAndExtract(DataSetType.TRAIN); }
+
+    /**
+     * Downloads and extracts the local dataset.
+     *
+     * @throws IOException
+     */
+    public void downloadAndExtract(DataSetType set) throws IOException {
+        String localFilename = new File(remoteDataUrl(set)).getName();
+        File tmpFile = new File(System.getProperty("java.io.tmpdir"), localFilename);
+        File localCacheDir = getLocalCacheDir();
+
+        // check empty cache
+        if(localCacheDir.exists()) {
+            File[] list = localCacheDir.listFiles();
+            if(list == null || list.length == 0)
+                localCacheDir.delete();
+        }
+
+        File localDestinationDir = new File(localCacheDir, dataSetName(set));
+        if(!localDestinationDir.exists()) {
+            localCacheDir.mkdirs();
+            tmpFile.delete();
+            log.info("Downloading dataset to " + tmpFile.getAbsolutePath());
+            FileUtils.copyURLToFile(new URL(remoteDataUrl(set)), tmpFile);
+        } else {
+            //Directory exists and is non-empty - assume OK
+            log.info("Using cached dataset at " + localCacheDir.getAbsolutePath());
+            return;
+        }
+
+        if(expectedChecksum(set) != 0L) {
+            log.info("Verifying download...");
+            Checksum adler = new Adler32();
+            FileUtils.checksum(tmpFile, adler);
+            long localChecksum = adler.getValue();
+            log.info("Checksum local is " + localChecksum + ", expecting "+expectedChecksum(set));
+
+            if(expectedChecksum(set) != localChecksum) {
+                log.error("Checksums do not match. Cleaning up files and failing...");
+                tmpFile.delete();
+                throw new IllegalStateException( "Dataset file failed checksum: " + tmpFile + " - expected checksum " + expectedChecksum(set)
+                + " vs. actual checksum " + localChecksum + ". If this error persists, please open an issue at https://github.com/eclipse/deeplearning4j.");
+            }
+        }
+
+        try {
+            ArchiveUtils.unzipFileTo(tmpFile.getAbsolutePath(), localCacheDir.getAbsolutePath(), false);
+        } catch (Throwable t){
+            //Catch any errors during extraction, and delete the directory to avoid leaving the dir in an invalid state
+            if(localCacheDir.exists())
+                FileUtils.deleteDirectory(localCacheDir);
+            throw t;
+        }
+    }
+
+    protected File getLocalCacheDir(){
+        return DL4JResources.getDirectory(ResourceType.DATASET, localCacheName());
+    }
+
+    /**
+     * Returns a boolean indicating if the dataset is already cached locally.
+     *
+     * @return boolean
+     */
+    @Override
+    public boolean isCached() {
+        return getLocalCacheDir().exists();
+    }
+
+
+    protected static void deleteIfEmpty(File localCache){
+        if(localCache.exists()) {
+            File[] files = localCache.listFiles();
+            if(files == null || files.length < 1){
+                try {
+                    FileUtils.deleteDirectory(localCache);
+                } catch (IOException e){
+                    //Ignore
+                    log.debug("Error deleting directory: {}", localCache);
+                }
+            }
+        }
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/Cifar10Fetcher.java

@@ -0,0 +1,104 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import org.datavec.api.io.filters.RandomPathFilter;
+import org.datavec.api.io.labels.ParentPathLabelGenerator;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.split.FileSplit;
+import org.datavec.api.split.InputSplit;
+import org.datavec.image.loader.BaseImageLoader;
+import org.datavec.image.recordreader.ImageRecordReader;
+import org.datavec.image.transform.ImageTransform;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.nd4j.common.base.Preconditions;
+
+import java.io.File;
+import java.util.Random;
+
+public class Cifar10Fetcher extends CacheableExtractableDataSetFetcher {
+    public static final String LABELS_FILENAME = "labels.txt";
+    public static final String LOCAL_CACHE_NAME = "cifar10";
+
+    public static int INPUT_WIDTH = 32;
+    public static int INPUT_HEIGHT = 32;
+    public static int INPUT_CHANNELS = 3;
+    public static int NUM_LABELS = 10;
+
+    @Override
+    public String remoteDataUrl(DataSetType set) {
+        return DL4JResources.getURLString("datasets/cifar10_dl4j.v1.zip");
+    }
+    @Override
+    public String localCacheName(){ return LOCAL_CACHE_NAME; }
+    @Override
+    public long expectedChecksum(DataSetType set) { return 292852033L; }
+    @Override
+    public RecordReader getRecordReader(long rngSeed, int[] imgDim, DataSetType set, ImageTransform imageTransform) {
+        Preconditions.checkState(imgDim == null || imgDim.length == 2, "Invalid image dimensions: must be null or lenth 2. Got: %s", imgDim);
+        // check empty cache
+        File localCache = getLocalCacheDir();
+        deleteIfEmpty(localCache);
+
+        try {
+            if (!localCache.exists()){
+                downloadAndExtract();
+            }
+        } catch(Exception e) {
+            throw new RuntimeException("Could not download CIFAR-10", e);
+        }
+
+        Random rng = new Random(rngSeed);
+        File datasetPath;
+        switch (set) {
+            case TRAIN:
+                datasetPath = new File(localCache, "/train/");
+                break;
+            case TEST:
+                datasetPath = new File(localCache, "/test/");
+                break;
+            case VALIDATION:
+                throw new IllegalArgumentException("You will need to manually create and iterate a validation directory, CIFAR-10 does not provide labels");
+
+            default:
+                datasetPath = new File(localCache, "/train/");
+        }
+
+        // set up file paths
+        RandomPathFilter pathFilter = new RandomPathFilter(rng, BaseImageLoader.ALLOWED_FORMATS);
+        FileSplit filesInDir = new FileSplit(datasetPath, BaseImageLoader.ALLOWED_FORMATS, rng);
+        InputSplit[] filesInDirSplit = filesInDir.sample(pathFilter, 1);
+
+        int h = (imgDim == null ? Cifar10Fetcher.INPUT_HEIGHT : imgDim[0]);
+        int w = (imgDim == null ? Cifar10Fetcher.INPUT_WIDTH : imgDim[1]);
+        ImageRecordReader rr = new ImageRecordReader(h, w, Cifar10Fetcher.INPUT_CHANNELS, new ParentPathLabelGenerator(), imageTransform);
+
+        try {
+            rr.initialize(filesInDirSplit[0]);
+        } catch(Exception e) {
+            throw new RuntimeException(e);
+        }
+
+        return rr;
+    }
+
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/DataSetType.java

@@ -0,0 +1,25 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+public enum DataSetType {
+    TRAIN, TEST, VALIDATION
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/EmnistDataFetcher.java

@@ -0,0 +1,115 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.deeplearning4j.datasets.base.EmnistFetcher;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.eclipse.deeplearning4j.resources.utils.EMnistSet;
+import org.deeplearning4j.datasets.iterator.impl.EmnistDataSetIterator;
+import org.deeplearning4j.datasets.mnist.MnistManager;
+import org.nd4j.linalg.dataset.api.iterator.fetcher.DataSetFetcher;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+
+
+@Slf4j
+public class EmnistDataFetcher extends MnistDataFetcher implements DataSetFetcher {
+
+    protected EmnistFetcher fetcher;
+
+
+
+    public EmnistDataFetcher(EMnistSet dataSet, boolean binarize, boolean train, boolean shuffle,
+                             long rngSeed,File topLevelDir) throws IOException {
+        fetcher = new EmnistFetcher(dataSet,topLevelDir);
+        if (!emnistExists(fetcher)) {
+            fetcher.downloadAndUntar();
+        }
+
+
+        String EMNIST_ROOT = topLevelDir.getAbsolutePath();
+        if (train) {
+            images = fetcher.getEmnistDataTrain().localPath().getAbsolutePath();
+            labels = fetcher.getEmnistLabelsTrain().localPath().getAbsolutePath();
+            totalExamples = EmnistDataSetIterator.numExamplesTrain(dataSet);
+        } else {
+            images = fetcher.getEmnistDataTest().localPath().getAbsolutePath();
+            labels = fetcher.getEmnistLabelsTest().localPath().getAbsolutePath();
+            totalExamples = EmnistDataSetIterator.numExamplesTest(dataSet);
+        }
+        try {
+            manager = new MnistManager(images, labels, totalExamples);
+        } catch (Exception e) {
+            log.error("",e);
+            FileUtils.deleteDirectory(new File(EMNIST_ROOT));
+            new EmnistFetcher(dataSet).downloadAndUntar();
+            manager = new MnistManager(images, labels, totalExamples);
+        }
+
+        numOutcomes = EmnistDataSetIterator.numLabels(dataSet);
+        this.binarize = binarize;
+        cursor = 0;
+        manager.setCurrent(cursor);
+        inputColumns = manager.getImages().getEntryLength();
+        this.train = train;
+        this.shuffle = shuffle;
+
+        order = new int[totalExamples];
+        for (int i = 0; i < order.length; i++)
+            order[i] = i;
+        rng = new Random(rngSeed);
+        reset(); //Shuffle order
+
+
+        //For some inexplicable reason, EMNIST LETTERS set is indexed 1 to 26 (i.e., 1 to nClasses), while everything else
+        // is indexed (0 to nClasses-1) :/
+        if (dataSet == EMnistSet.LETTERS) {
+            oneIndexed = true;
+        } else {
+            oneIndexed = false;
+        }
+        this.fOrder = true; //MNIST is C order, EMNIST is F order
+    }
+
+    public EmnistDataFetcher(EMnistSet dataSet, boolean binarize, boolean train, boolean shuffle,
+                             long rngSeed) throws IOException {
+        this(dataSet,binarize,train,shuffle,rngSeed,DL4JResources.getDirectory(ResourceType.DATASET, "EMNIST"));
+    }
+
+    private boolean emnistExists(EmnistFetcher e) {
+        //Check 4 files:
+        if (!fetcher.getEmnistDataTrain().existsLocally())
+            return false;
+        if (!fetcher.getEmnistLabelsTrain().existsLocally())
+            return false;
+        if (!fetcher.getEmnistDataTest().existsLocally())
+            return false;
+        if (!fetcher.getEmnistLabelsTest().existsLocally())
+            return false;
+        return true;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/IrisDataFetcher.java

@@ -0,0 +1,60 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import org.deeplearning4j.datasets.base.IrisUtils;
+import org.nd4j.linalg.dataset.api.iterator.fetcher.BaseDataFetcher;
+
+import java.io.IOException;
+
+
+public class IrisDataFetcher extends BaseDataFetcher {
+
+
+    /**
+     * 
+     */
+    private static final long serialVersionUID = 4566329799221375262L;
+    public final static int NUM_EXAMPLES = 150;
+
+    public IrisDataFetcher() {
+        numOutcomes = 3;
+        inputColumns = 4;
+        totalExamples = NUM_EXAMPLES;
+    }
+
+    @Override
+    public void fetch(int numExamples) {
+        int from = cursor;
+        int to = cursor + numExamples;
+        if (to > totalExamples)
+            to = totalExamples;
+
+        try {
+            initializeCurrFromList(IrisUtils.loadIris(from, to));
+            cursor += numExamples;
+        } catch (IOException e) {
+            throw new IllegalStateException("Unable to load iris.dat", e);
+        }
+    }
+
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/MnistDataFetcher.java

@@ -0,0 +1,278 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import lombok.SneakyThrows;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.deeplearning4j.datasets.base.MnistFetcher;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.deeplearning4j.datasets.mnist.MnistManager;
+import org.eclipse.deeplearning4j.resources.DataSetResource;
+import org.eclipse.deeplearning4j.resources.ResourceDataSets;
+import org.nd4j.linalg.api.buffer.DataType;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.iterator.fetcher.BaseDataFetcher;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.indexing.NDArrayIndex;
+import org.nd4j.common.util.MathUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.zip.Adler32;
+import java.util.zip.Checksum;
+
+
+public class MnistDataFetcher extends BaseDataFetcher {
+    public static final int NUM_EXAMPLES = 60000;
+    public static final int NUM_EXAMPLES_TEST = 10000;
+
+    protected static final long CHECKSUM_TRAIN_FEATURES = 2094436111L;
+    protected static final long CHECKSUM_TRAIN_LABELS = 4008842612L;
+    protected static final long CHECKSUM_TEST_FEATURES = 2165396896L;
+    protected static final long CHECKSUM_TEST_LABELS = 2212998611L;
+
+    protected static final long[] CHECKSUMS_TRAIN = new long[]{CHECKSUM_TRAIN_FEATURES, CHECKSUM_TRAIN_LABELS};
+    protected static final long[] CHECKSUMS_TEST = new long[]{CHECKSUM_TEST_FEATURES, CHECKSUM_TEST_LABELS};
+
+    protected boolean binarize = true;
+    protected boolean train;
+    protected int[] order;
+    protected Random rng;
+    protected boolean shuffle;
+    protected boolean oneIndexed = false;
+    protected boolean fOrder = false; //MNIST is C order, EMNIST is F order
+
+    protected boolean firstShuffle = true;
+    protected  int numExamples = 0;
+    protected String images,labels;
+    //note: we default to zero here on purpose, otherwise when first initializes an error is thrown.
+    private long lastCursor = 0;
+    protected MnistManager manager;
+
+    /**
+     * Constructor telling whether to binarize the dataset or not
+     * @param binarize whether to binarize the dataset or not
+     * @throws IOException
+     */
+    public MnistDataFetcher(boolean binarize) throws IOException {
+        this(binarize, true, true, System.currentTimeMillis(), NUM_EXAMPLES);
+    }
+
+
+
+    public MnistDataFetcher(boolean binarize, boolean train, boolean shuffle, long rngSeed, int numExamples,File topLevelDir) throws IOException {
+        if(this instanceof EmnistDataFetcher)
+            return;
+
+
+        this.topLevelDir = topLevelDir;
+        long[] checksums;
+        DataSetResource imageResource = null;
+        DataSetResource labelResource = null;
+        if (train) {
+            imageResource = topLevelDir() != null ?  ResourceDataSets.mnistTrain(topLevelDir()) :  ResourceDataSets.mnistTrain();
+            if(!imageResource.existsLocally())
+                imageResource.download(true,3,200000,20000);
+
+            labelResource = topLevelDir() != null ? ResourceDataSets.mnistTrainLabels(topLevelDir()) : ResourceDataSets.mnistTrainLabels();
+            if(!labelResource.existsLocally())
+                labelResource.download(true,3,200000,20000);
+
+            totalExamples = NUM_EXAMPLES;
+            checksums = CHECKSUMS_TRAIN;
+        } else {
+            imageResource = topLevelDir() != null ?  ResourceDataSets.mnistTest(topLevelDir()) :  ResourceDataSets.mnistTest();
+            if(!imageResource.existsLocally())
+                imageResource.download(true,3,200000,20000);
+
+            labelResource = topLevelDir() != null ? ResourceDataSets.mnistTestLabels(topLevelDir()) : ResourceDataSets.mnistTestLabels();
+            if(!labelResource.existsLocally())
+                labelResource.download(true,3,200000,20000);
+
+            totalExamples = NUM_EXAMPLES_TEST;
+            checksums = CHECKSUMS_TEST;
+        }
+
+        images = imageResource.localPath().getAbsolutePath();
+        labels = labelResource.localPath().getAbsolutePath();
+
+
+        try {
+            manager = new MnistManager(images, labels, train);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+
+
+        numOutcomes = 10;
+        this.binarize = binarize;
+        cursor = 0;
+        inputColumns = manager.getImages().getEntryLength();
+        this.train = train;
+        this.shuffle = shuffle;
+
+        if (train) {
+            order = new int[NUM_EXAMPLES];
+        } else {
+            order = new int[NUM_EXAMPLES_TEST];
+        }
+        for (int i = 0; i < order.length; i++)
+            order[i] = i;
+        rng = new Random(rngSeed);
+        this.numExamples = numExamples;
+        reset(); //Shuffle order
+    }
+
+    public MnistDataFetcher(boolean binarize, boolean train, boolean shuffle, long rngSeed, int numExamples) throws IOException {
+        this(binarize,train,shuffle,rngSeed,numExamples,null);
+    }
+
+
+
+    private void validateFiles(String[] files, long[] checksums) {
+        //Validate files:
+        try {
+            for (int i = 0; i < files.length; i++) {
+                File f = new File(files[i]);
+                Checksum adler = new Adler32();
+                long checksum = f.exists() ? FileUtils.checksum(f, adler).getValue() : -1;
+                if (!f.exists() || checksum != checksums[i]) {
+                    throw new IllegalStateException("Failed checksum: expected " + checksums[i] +
+                            ", got " + checksum + " for file: " + f);
+                }
+            }
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public MnistDataFetcher() throws IOException {
+        this(true);
+    }
+
+    private float[][] featureData = null;
+
+    @SneakyThrows
+    @Override
+    public void fetch(int numExamples) {
+        if (!hasMore()) {
+            throw new IllegalStateException("Unable to get more; there are no more images");
+        }
+
+        manager.setCurrent((int) lastCursor);
+        INDArray labels = Nd4j.zeros(DataType.FLOAT, numExamples, numOutcomes);
+
+        if(featureData == null || featureData.length < numExamples){
+            featureData = new float[numExamples][28 * 28];
+        }
+
+        int actualExamples = 0;
+        byte[] working = null;
+        for (int i = 0; i < numExamples; i++, cursor++) {
+            if (!hasMore())
+                break;
+
+            manager.setCurrent(cursor);
+            lastCursor = cursor;
+            byte[] img = manager.readImageUnsafe(order[cursor]);
+
+            if (fOrder) {
+                //EMNIST requires F order to C order
+                if (working == null) {
+                    working = new byte[28 * 28];
+                }
+                for (int j = 0; j < 28 * 28; j++) {
+                    working[j] = img[28 * (j % 28) + j / 28];
+                }
+                img = working;
+            }
+
+            int label = manager.readLabel(order[cursor]);
+            if (oneIndexed) {
+                //For some inexplicable reason, Emnist LETTERS set is indexed 1 to 26 (i.e., 1 to nClasses), while everything else
+                // is indexed (0 to nClasses-1) :/
+                label--;
+            }
+
+            labels.put(actualExamples, label, 1.0f);
+
+            for(int j = 0 ; j < img.length ; j++) {
+                featureData[actualExamples][j] = ((int) img[j]) & 0xFF;
+            }
+
+            actualExamples++;
+        }
+
+        INDArray features;
+
+        if(featureData.length == actualExamples){
+            features = Nd4j.create(featureData);
+        } else {
+            features = Nd4j.create(Arrays.copyOfRange(featureData, 0, actualExamples));
+        }
+
+        if (actualExamples < numExamples) {
+            labels = labels.get(NDArrayIndex.interval(0, actualExamples), NDArrayIndex.all());
+        }
+
+        if(binarize){
+            features = features.gt(30.0).castTo(DataType.FLOAT);
+        } else {
+            features.divi(255.0);
+        }
+
+        curr = new DataSet(features, labels);
+    }
+
+    @Override
+    public void reset() {
+        cursor = 0;
+        curr = null;
+        if (shuffle) {
+            if((train && numExamples < NUM_EXAMPLES) || (!train && numExamples < NUM_EXAMPLES_TEST)){
+                //Shuffle only first N elements
+                if(firstShuffle){
+                    MathUtils.shuffleArray(order, rng);
+                    firstShuffle = false;
+                } else {
+                    MathUtils.shuffleArraySubset(order, numExamples, rng);
+                }
+            } else {
+                MathUtils.shuffleArray(order, rng);
+            }
+        }
+    }
+
+    @Override
+    public DataSet next() {
+        DataSet next = super.next();
+        return next;
+    }
+
+    public void close() {
+    }
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcher.java

@@ -0,0 +1,132 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.split.FileSplit;
+import org.datavec.image.loader.BaseImageLoader;
+import org.datavec.image.recordreader.objdetect.ObjectDetectionRecordReader;
+import org.datavec.image.transform.ImageTransform;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+
+public class SvhnDataFetcher extends CacheableExtractableDataSetFetcher {
+
+    private static String BASE_URL = "http://ufldl.stanford.edu/";
+
+    public static void setBaseUrl(String baseUrl){
+        BASE_URL = baseUrl;
+    }
+
+    public static int NUM_LABELS = 10;
+
+    @Override
+    public String remoteDataUrl(DataSetType set) {
+        switch (set) {
+            case TRAIN:
+                return BASE_URL + "housenumbers/train.tar.gz";
+            case TEST:
+                return BASE_URL + "housenumbers/test.tar.gz";
+            case VALIDATION:
+                return BASE_URL + "housenumbers/extra.tar.gz";
+            default:
+                 throw new IllegalArgumentException("Unknown DataSetType:" + set);
+        }
+    }
+
+    @Override
+    public String localCacheName() {
+        return "SVHN";
+    }
+
+    @Override
+    public String dataSetName(DataSetType set) {
+        switch (set) {
+            case TRAIN:
+                return "train";
+            case TEST:
+                return "test";
+            case VALIDATION:
+                return "extra";
+            default:
+                throw new IllegalArgumentException("Unknown DataSetType:" + set);
+        }
+    }
+
+    @Override
+    public long expectedChecksum(DataSetType set) {
+        switch (set) {
+            case TRAIN:
+                return 979655493L;
+            case TEST:
+                return 1629515343L;
+            case VALIDATION:
+                return 132781169L;
+            default:
+                 throw new IllegalArgumentException("Unknown DataSetType:" + set);
+        }
+    }
+
+    public File getDataSetPath(DataSetType set) throws IOException {
+        File localCache = getLocalCacheDir();
+        // check empty cache
+        deleteIfEmpty(localCache);
+
+        File datasetPath;
+        switch (set) {
+            case TRAIN:
+                datasetPath = new File(localCache, "/train/");
+                break;
+            case TEST:
+                datasetPath = new File(localCache, "/test/");
+                break;
+            case VALIDATION:
+                datasetPath = new File(localCache, "/extra/");
+                break;
+            default:
+                datasetPath = null;
+        }
+
+        if (!datasetPath.exists()) {
+            downloadAndExtract(set);
+        }
+        return datasetPath;
+    }
+
+    @Override
+    public RecordReader getRecordReader(long rngSeed, int[] imgDim, DataSetType set, ImageTransform imageTransform) {
+        try {
+            Random rng = new Random(rngSeed);
+            File datasetPath = getDataSetPath(set);
+
+            FileSplit data = new FileSplit(datasetPath, BaseImageLoader.ALLOWED_FORMATS, rng);
+            ObjectDetectionRecordReader recordReader = new ObjectDetectionRecordReader(imgDim[1], imgDim[0], imgDim[2],
+                            imgDim[4], imgDim[3], null);
+
+            recordReader.initialize(data);
+            return recordReader;
+        } catch (IOException e) {
+            throw new RuntimeException("Could not download SVHN", e);
+        }
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/TinyImageNetFetcher.java

@@ -0,0 +1,105 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import org.datavec.api.io.filters.RandomPathFilter;
+import org.datavec.api.io.labels.ParentPathLabelGenerator;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.split.FileSplit;
+import org.datavec.api.split.InputSplit;
+import org.datavec.image.loader.BaseImageLoader;
+import org.datavec.image.recordreader.ImageRecordReader;
+import org.datavec.image.transform.ImageTransform;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.nd4j.common.base.Preconditions;
+
+import java.io.File;
+import java.util.Random;
+
+public class TinyImageNetFetcher extends CacheableExtractableDataSetFetcher {
+    public static final String WORDS_FILENAME = "words.txt";
+    public static final String LOCAL_CACHE_NAME = "TINYIMAGENET_200";
+
+    public static int INPUT_WIDTH = 64;
+    public static int INPUT_HEIGHT = 64;
+    public static int INPUT_CHANNELS = 3;
+    public static int NUM_LABELS = 200;
+    public static int NUM_EXAMPLES = NUM_LABELS*500;
+
+    @Override
+    public String remoteDataUrl(DataSetType set) {
+        return DL4JResources.getURLString("datasets/tinyimagenet_200_dl4j.v1.zip");
+    }
+    @Override
+    public String localCacheName(){ return LOCAL_CACHE_NAME; }
+    @Override
+    public long expectedChecksum(DataSetType set) { return 33822361L; }
+    @Override
+    public RecordReader getRecordReader(long rngSeed, int[] imgDim, DataSetType set, ImageTransform imageTransform) {
+        Preconditions.checkState(imgDim == null || imgDim.length == 2, "Invalid image dimensions: must be null or lenth 2. Got: %s", imgDim);
+        // check empty cache
+        File localCache = getLocalCacheDir();
+        deleteIfEmpty(localCache);
+
+        try {
+            if (!localCache.exists()){
+                downloadAndExtract();
+            }
+        } catch(Exception e) {
+            throw new RuntimeException("Could not download TinyImageNet", e);
+        }
+
+        Random rng = new Random(rngSeed);
+        File datasetPath;
+        switch (set) {
+            case TRAIN:
+                datasetPath = new File(localCache, "/train/");
+                break;
+            case TEST:
+                datasetPath = new File(localCache, "/test/");
+                break;
+            case VALIDATION:
+                throw new IllegalArgumentException("You will need to manually iterate the /validation/images/ directory, TinyImageNet does not provide labels");
+
+            default:
+                datasetPath = new File(localCache, "/train/");
+        }
+
+        // set up file paths
+        RandomPathFilter pathFilter = new RandomPathFilter(rng, BaseImageLoader.ALLOWED_FORMATS);
+        FileSplit filesInDir = new FileSplit(datasetPath, BaseImageLoader.ALLOWED_FORMATS, rng);
+        InputSplit[] filesInDirSplit = filesInDir.sample(pathFilter, 1);
+
+        int h = (imgDim == null ? TinyImageNetFetcher.INPUT_HEIGHT : imgDim[0]);
+        int w = (imgDim == null ? TinyImageNetFetcher.INPUT_WIDTH : imgDim[1]);
+        ImageRecordReader rr = new ImageRecordReader(h, w,TinyImageNetFetcher.INPUT_CHANNELS, new ParentPathLabelGenerator(), imageTransform);
+
+        try {
+            rr.initialize(filesInDirSplit[0]);
+        } catch(Exception e) {
+            throw new RuntimeException(e);
+        }
+
+        return rr;
+    }
+
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/UciSequenceDataFetcher.java

@@ -0,0 +1,164 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.fetchers;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader;
+import org.datavec.api.split.NumberedFileInputSplit;
+import org.datavec.image.transform.ImageTransform;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Random;
+
+@Slf4j
+public class UciSequenceDataFetcher extends CacheableExtractableDataSetFetcher {
+
+    public static int NUM_LABELS = 6;
+    public static int NUM_EXAMPLES = NUM_LABELS * 100;
+    private static String url = "https://archive.ics.uci.edu/ml/machine-learning-databases/synthetic_control-mld/synthetic_control.data";
+
+    public static void setURL(String url){
+        UciSequenceDataFetcher.url = url;
+    }
+
+    @Override
+    public String remoteDataUrl() {
+        return url;
+    }
+
+    @Override
+    public String remoteDataUrl(DataSetType type) {
+        return remoteDataUrl();
+    }
+
+    @Override
+    public String localCacheName() {
+        return "UCISequence_6";
+    }
+
+    @Override
+    public long expectedChecksum() {
+        return 104392751L;
+    }
+
+    @Override
+    public long expectedChecksum(DataSetType type) {
+        return expectedChecksum();
+    }
+
+    @Override
+    public CSVSequenceRecordReader getRecordReader(long rngSeed, int[] shape, DataSetType set, ImageTransform transform) {
+        return getRecordReader(rngSeed, set);
+    }
+
+    public CSVSequenceRecordReader getRecordReader(long rngSeed, DataSetType set) {
+
+        // check empty cache
+        File localCache = getLocalCacheDir();
+        deleteIfEmpty(localCache);
+
+        try {
+            if (!localCache.exists()) downloadAndExtract();
+        } catch (Exception e) {
+            throw new RuntimeException("Could not download UCI Sequence data", e);
+        }
+
+        File dataPath;
+
+        switch (set) {
+            case TRAIN:
+                dataPath = new File(localCache, "/train");
+                break;
+            case TEST:
+                dataPath = new File(localCache, "/test");
+                break;
+            case VALIDATION:
+                throw new IllegalArgumentException("You will need to manually iterate the directory, UCISequence data does not provide labels");
+
+            default:
+                dataPath = new File(localCache, "/train");
+        }
+
+        try {
+            downloadUCIData(dataPath);
+            CSVSequenceRecordReader data;
+            switch (set) {
+                case TRAIN:
+                    data = new CSVSequenceRecordReader(0, ", ");
+                    data.initialize(new NumberedFileInputSplit(dataPath.getAbsolutePath() + "/%d.csv", 0, 449));
+                    break;
+                case TEST:
+                    data = new CSVSequenceRecordReader(0, ", ");
+                    data.initialize(new NumberedFileInputSplit(dataPath.getAbsolutePath() + "/%d.csv", 450, 599));
+                    break;
+                default:
+                    data = new CSVSequenceRecordReader(0, ", ");
+                    data.initialize(new NumberedFileInputSplit(dataPath.getAbsolutePath() + "/%d.csv", 0, 449));
+            }
+
+            return data;
+        } catch (Exception e) {
+            throw new RuntimeException("Could not process UCI data", e);
+        }
+    }
+
+    private static void downloadUCIData(File dataPath) throws Exception {
+        //if (dataPath.exists()) return;
+
+        String data = IOUtils.toString(new URL(url), Charset.defaultCharset());
+        String[] lines = data.split("\n");
+
+        int lineCount = 0;
+        int index = 0;
+
+        ArrayList<String> linesList = new ArrayList<>();
+
+        for (String line : lines) {
+
+            // label value
+            int count = lineCount++ / 100;
+
+            // replace white space with commas and label value + new line
+            line = line.replaceAll("\\s+", ", " + count + "\n");
+
+            // add label to last number
+            line = line + ", " + count;
+            linesList.add(line);
+        }
+
+        // randomly shuffle data
+        Collections.shuffle(linesList, new Random(12345));
+
+        for (String line : linesList) {
+            File outPath = new File(dataPath, index + ".csv");
+            FileUtils.writeStringToFile(outPath, line, Charset.defaultCharset());
+            index++;
+        }
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/Cifar10DataSetIterator.java

@@ -0,0 +1,127 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.impl;
+
+import lombok.Getter;
+import org.apache.commons.io.FileUtils;
+import org.datavec.image.transform.ImageTransform;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator;
+import org.deeplearning4j.datasets.fetchers.Cifar10Fetcher;
+import org.deeplearning4j.datasets.fetchers.DataSetType;
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class Cifar10DataSetIterator extends RecordReaderDataSetIterator {
+
+    @Getter
+    protected DataSetPreProcessor preProcessor;
+
+    /**
+     * Create an iterator for the training set, with random iteration order (RNG seed fixed to 123)
+     *
+     * @param batchSize Minibatch size for the iterator
+     */
+    public Cifar10DataSetIterator(int batchSize) {
+        this(batchSize, null, DataSetType.TRAIN, null, 123);
+    }
+
+    /**
+     * * Create an iterator for the training or test set, with random iteration order (RNG seed fixed to 123)
+     *
+     * @param batchSize Minibatch size for the iterator
+     * @param set       The dataset (train or test)
+     */
+    public Cifar10DataSetIterator(int batchSize, DataSetType set) {
+        this(batchSize, null, set, null, 123);
+    }
+
+    /**
+     * Get the Tiny ImageNet iterator with specified train/test set, with random iteration order (RNG seed fixed to 123)
+     *
+     * @param batchSize Size of each patch
+     * @param imgDim Dimensions of desired output - for example, {64, 64}
+     * @param set Train, test, or validation
+     */
+    public Cifar10DataSetIterator(int batchSize, int[] imgDim, DataSetType set) {
+        this(batchSize, imgDim, set, null, 123);
+    }
+
+    /**
+     * Get the Tiny ImageNet iterator with specified train/test set and (optional) custom transform.
+     *
+     * @param batchSize Size of each patch
+     * @param imgDim Dimensions of desired output - for example, {64, 64}
+     * @param set Train, test, or validation
+     * @param imageTransform Additional image transform for output
+     * @param rngSeed random number generator seed to use when shuffling examples
+     */
+    public Cifar10DataSetIterator(int batchSize, int[] imgDim, DataSetType set,
+                                  ImageTransform imageTransform, long rngSeed) {
+        super(new Cifar10Fetcher().getRecordReader(rngSeed, imgDim, set, imageTransform), batchSize, 1, Cifar10Fetcher.NUM_LABELS);
+    }
+
+    /**
+     * Get the labels - either in "categories" (imagenet synsets format, "n01910747" or similar) or human-readable format,
+     * such as "jellyfish"
+     * @param categories If true: return category/synset format; false: return "human readable" label format
+     * @return Labels
+     */
+    public static List<String> getLabels(boolean categories){
+        List<String> rawLabels = new Cifar10DataSetIterator(1).getLabels();
+        if(categories){
+            return rawLabels;
+        }
+
+        //Otherwise, convert to human-readable format, using 'words.txt' file
+        File baseDir = DL4JResources.getDirectory(ResourceType.DATASET, Cifar10Fetcher.LOCAL_CACHE_NAME);
+        File labelFile = new File(baseDir, Cifar10Fetcher.LABELS_FILENAME);
+        List<String> lines;
+        try {
+            lines = FileUtils.readLines(labelFile, StandardCharsets.UTF_8);
+        } catch (IOException e){
+            throw new RuntimeException("Error reading label file", e);
+        }
+
+        Map<String,String> map = new HashMap<>();
+        for(String line : lines){
+            String[] split = line.split("\t");
+            map.put(split[0], split[1]);
+        }
+
+        List<String> outLabels = new ArrayList<>(rawLabels.size());
+        for(String s : rawLabels){
+            String s2 = map.get(s);
+            Preconditions.checkState(s2 != null, "Label \"%s\" not found in labels.txt file");
+            outLabels.add(s2);
+        }
+        return outLabels;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/EmnistDataSetIterator.java

@@ -0,0 +1,295 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.impl;
+
+import lombok.Getter;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.deeplearning4j.datasets.fetchers.EmnistDataFetcher;
+import org.eclipse.deeplearning4j.resources.utils.EMnistSet;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+import org.nd4j.linalg.dataset.api.iterator.BaseDatasetIterator;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class EmnistDataSetIterator extends BaseDatasetIterator {
+
+    private static final int NUM_COMPLETE_TRAIN = 697932;
+    private static final int NUM_COMPLETE_TEST = 116323;
+
+    private static final int NUM_MERGE_TRAIN = 697932;
+    private static final int NUM_MERGE_TEST = 116323;
+
+    private static final int NUM_BALANCED_TRAIN = 112800;
+    private static final int NUM_BALANCED_TEST = 18800;
+
+    private static final int NUM_DIGITS_TRAIN = 240000;
+    private static final int NUM_DIGITS_TEST = 40000;
+
+    private static final int NUM_LETTERS_TRAIN = 88800;
+    private static final int NUM_LETTERS_TEST = 14800;
+
+    private static final int NUM_MNIST_TRAIN = 60000;
+    private static final int NUM_MNIST_TEST = 10000;
+
+    private static final char[] LABELS_COMPLETE = new char[] {48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 65, 66, 67, 68,
+            69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99,
+            100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
+            120, 121, 122};
+
+    private static final char[] LABELS_MERGE = new char[] {48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 65, 66, 67, 68, 69,
+            70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 100,
+            101, 102, 103, 104, 110, 113, 114, 116};
+
+    private static final char[] LABELS_BALANCED = new char[] {48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 65, 66, 67, 68,
+            69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 100,
+            101, 102, 103, 104, 110, 113, 114, 116};
+
+    private static final char[] LABELS_DIGITS = new char[] {48, 49, 50, 51, 52, 53, 54, 55, 56, 57};
+
+    private static final char[] LABELS_LETTERS = new char[] {65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
+            80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90};
+
+    protected EMnistSet dataSet;
+    protected int batch, numExamples;
+    @Getter
+    protected DataSetPreProcessor preProcessor;
+
+    /**
+     * Create an EMNIST iterator with randomly shuffled data based on a random RNG seed
+     *
+     * @param dataSet Dataset (subset) to return
+     * @param batch   Batch size
+     * @param train   If true: use training set. If false: use test set
+     * @throws IOException If an error occurs when loading/downloading the dataset
+     */
+    public EmnistDataSetIterator(EMnistSet dataSet, int batch, boolean train) throws IOException {
+        this(dataSet, batch, train, System.currentTimeMillis());
+    }
+
+    /**
+     * Create an EMNIST iterator with randomly shuffled data based on a specified RNG seed
+     *
+     * @param dataSet   Dataset (subset) to return
+     * @param batchSize Batch size
+     * @param train     If true: use training set. If false: use test set
+     * @param seed      Random number generator seed
+     */
+    public EmnistDataSetIterator(EMnistSet dataSet, int batchSize, boolean train, long seed) throws IOException {
+        this(dataSet, batchSize, false, train, true, seed);
+    }
+
+    /**
+     * Get the specified number of MNIST examples (test or train set), with optional shuffling and binarization.
+     *
+     * @param batch    Size of each minibatch
+     * @param binarize whether to binarize the data or not (if false: normalize in range 0 to 1)
+     * @param train    Train vs. test set
+     * @param shuffle  whether to shuffle the examples
+     * @param rngSeed  random number generator seed to use when shuffling examples
+     */
+    public EmnistDataSetIterator(EMnistSet dataSet, int batch, boolean binarize, boolean train, boolean shuffle, long rngSeed, File topLevelDir)
+            throws IOException {
+        super(batch, numExamples(train, dataSet), new EmnistDataFetcher(dataSet, binarize, train, shuffle, rngSeed));
+        this.dataSet = dataSet;
+    }
+
+
+    /**
+     * Get the specified number of MNIST examples (test or train set), with optional shuffling and binarization.
+     *
+     * @param batch    Size of each minibatch
+     * @param binarize whether to binarize the data or not (if false: normalize in range 0 to 1)
+     * @param train    Train vs. test set
+     * @param shuffle  whether to shuffle the examples
+     * @param rngSeed  random number generator seed to use when shuffling examples
+     */
+    public EmnistDataSetIterator(EMnistSet dataSet, int batch, boolean binarize, boolean train, boolean shuffle, long rngSeed)
+            throws IOException {
+        this(dataSet,batch,binarize,train,shuffle,rngSeed, DL4JResources.getDirectory(ResourceType.DATASET,"emnist"));
+    }
+
+    private static int numExamples(boolean train, EMnistSet ds) {
+        if (train) {
+            return numExamplesTrain(ds);
+        } else {
+            return numExamplesTest(ds);
+        }
+    }
+
+    /**
+     * Get the number of training examples for the specified subset
+     *
+     * @param dataSet Subset to get
+     * @return Number of examples for the specified subset
+     */
+    public static int numExamplesTrain(EMnistSet dataSet) {
+        switch (dataSet) {
+            case COMPLETE:
+                return NUM_COMPLETE_TRAIN;
+            case MERGE:
+                return NUM_MERGE_TRAIN;
+            case BALANCED:
+                return NUM_BALANCED_TRAIN;
+            case LETTERS:
+                return NUM_LETTERS_TRAIN;
+            case DIGITS:
+                return NUM_DIGITS_TRAIN;
+            case MNIST:
+                return NUM_MNIST_TRAIN;
+            default:
+                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
+        }
+    }
+
+    /**
+     * Get the number of test examples for the specified subset
+     *
+     * @param dataSet Subset to get
+     * @return Number of examples for the specified subset
+     */
+    public static int numExamplesTest(EMnistSet dataSet) {
+        switch (dataSet) {
+            case COMPLETE:
+                return NUM_COMPLETE_TEST;
+            case MERGE:
+                return NUM_MERGE_TEST;
+            case BALANCED:
+                return NUM_BALANCED_TEST;
+            case LETTERS:
+                return NUM_LETTERS_TEST;
+            case DIGITS:
+                return NUM_DIGITS_TEST;
+            case MNIST:
+                return NUM_MNIST_TEST;
+            default:
+                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
+        }
+    }
+
+    /**
+     * Get the number of labels for the specified subset
+     *
+     * @param dataSet Subset to get
+     * @return Number of labels for the specified subset
+     */
+    public static int numLabels(EMnistSet dataSet) {
+        switch (dataSet) {
+            case COMPLETE:
+                return 62;
+            case MERGE:
+                return 47;
+            case BALANCED:
+                return 47;
+            case LETTERS:
+                return 26;
+            case DIGITS:
+                return 10;
+            case MNIST:
+                return 10;
+            default:
+                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
+        }
+    }
+
+    /**
+     * Get the labels as a character array
+     *
+     * @return Labels
+     */
+    public char[] getLabelsArrays() {
+        return getLabelsArray(dataSet);
+    }
+
+    /**
+     * Get the labels as a List<String>
+     *
+     * @return Labels
+     */
+    public List<String> getLabels() {
+        return getLabels(dataSet);
+    }
+
+    /**
+     * Get the label assignments for the given set as a character array.
+     *
+     * @param dataSet DataSet to get the label assignment for
+     * @return Label assignment and given dataset
+     */
+    public static char[] getLabelsArray(EMnistSet dataSet) {
+        switch (dataSet) {
+            case COMPLETE:
+                return LABELS_COMPLETE;
+            case MERGE:
+                return LABELS_MERGE;
+            case BALANCED:
+                return LABELS_BALANCED;
+            case LETTERS:
+                return LABELS_LETTERS;
+            case DIGITS:
+            case MNIST:
+                return LABELS_DIGITS;
+            default:
+                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
+        }
+    }
+
+    /**
+     * Get the label assignments for the given set as a List<String>
+     *
+     * @param dataSet DataSet to get the label assignment for
+     * @return Label assignment and given dataset
+     */
+    public static List<String> getLabels(EMnistSet dataSet) {
+        char[] c = getLabelsArray(dataSet);
+        List<String> l = new ArrayList<>(c.length);
+        for (char c2 : c) {
+            l.add(String.valueOf(c2));
+        }
+        return l;
+    }
+
+    /**
+     * Are the labels balanced in the training set (that is: are the number of examples for each label equal?)
+     *
+     * @param dataSet Set to get balanced value for
+     * @return True if balanced dataset, false otherwise
+     */
+    public static boolean isBalanced(EMnistSet dataSet) {
+        switch (dataSet) {
+            case COMPLETE:
+            case MERGE:
+            case LETTERS:
+                //Note: EMNIST docs claims letters is balanced, but this is not possible for training set:
+                // 88800 examples / 26 classes = 3418.46
+                return false;
+            case BALANCED:
+            case DIGITS:
+            case MNIST:
+                return true;
+            default:
+                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
+        }
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/IrisDataSetIterator.java

@@ -0,0 +1,63 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.impl;
+
+import org.deeplearning4j.datasets.fetchers.IrisDataFetcher;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.iterator.BaseDatasetIterator;
+
+public class IrisDataSetIterator extends BaseDatasetIterator {
+
+    /**
+     * 
+     */
+    private static final long serialVersionUID = -2022454995728680368L;
+
+    /**
+     * Create an iris iterator for full batch training - i.e., all 150 examples are included per minibatch
+     */
+    public IrisDataSetIterator(){
+        this(150, 150);
+    }
+
+    /**
+     * IrisDataSetIterator handles traversing through the Iris Data Set.
+     * @see <a href="https://archive.ics.uci.edu/ml/datasets/Iris">https://archive.ics.uci.edu/ml/datasets/Iris</a>
+     *
+     * @param batch Batch size
+     * @param numExamples Total number of examples
+     */
+    public IrisDataSetIterator(int batch, int numExamples) {
+        super(batch, numExamples, new IrisDataFetcher());
+    }
+
+
+    @Override
+    public DataSet next() {
+        fetcher.fetch(batch);
+        DataSet next = fetcher.next();
+        if(preProcessor != null) {
+            preProcessor.preProcess(next);
+        }
+
+        return next;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/LFWDataSetIterator.java

@@ -0,0 +1,106 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.impl;
+
+
+import org.datavec.api.io.labels.ParentPathLabelGenerator;
+import org.datavec.api.io.labels.PathLabelGenerator;
+import org.datavec.image.loader.LFWLoader;
+import org.datavec.image.transform.ImageTransform;
+import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator;
+
+import java.util.Random;
+
+public class LFWDataSetIterator extends RecordReaderDataSetIterator {
+
+    /** Loads subset of images with given imgDim returned by the generator. */
+    public LFWDataSetIterator(int[] imgDim) {
+        this(LFWLoader.SUB_NUM_IMAGES, LFWLoader.SUB_NUM_IMAGES, imgDim, LFWLoader.SUB_NUM_LABELS, false,
+                        new ParentPathLabelGenerator(), true, 1, null, new Random(System.currentTimeMillis()));
+    }
+
+    /** Loads images with given  batchSize, numExamples returned by the generator. */
+    public LFWDataSetIterator(int batchSize, int numExamples) {
+        this(batchSize, numExamples, new int[] {LFWLoader.HEIGHT, LFWLoader.WIDTH, LFWLoader.CHANNELS},
+                        LFWLoader.NUM_LABELS, false, LFWLoader.LABEL_PATTERN, true, 1, null,
+                        new Random(System.currentTimeMillis()));
+    }
+
+    /** Loads images with given  batchSize, numExamples, imgDim returned by the generator. */
+    public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim) {
+        this(batchSize, numExamples, imgDim, LFWLoader.NUM_LABELS, false, LFWLoader.LABEL_PATTERN, true, 1, null,
+                        new Random(System.currentTimeMillis()));
+    }
+
+    /** Loads images with given  batchSize, imgDim, useSubset, returned by the generator. */
+    public LFWDataSetIterator(int batchSize, int[] imgDim, boolean useSubset) {
+        this(batchSize, useSubset ? LFWLoader.SUB_NUM_IMAGES : LFWLoader.NUM_IMAGES, imgDim,
+                        useSubset ? LFWLoader.SUB_NUM_LABELS : LFWLoader.NUM_LABELS, useSubset, LFWLoader.LABEL_PATTERN,
+                        true, 1, null, new Random(System.currentTimeMillis()));
+    }
+
+    /** Loads images with given  batchSize, numExamples, imgDim, train, & splitTrainTest returned by the generator. */
+    public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, boolean train, double splitTrainTest) {
+        this(batchSize, numExamples, imgDim, LFWLoader.NUM_LABELS, false, LFWLoader.LABEL_PATTERN, train,
+                        splitTrainTest, null, new Random(System.currentTimeMillis()));
+    }
+
+    /** Loads images with given  batchSize, numExamples, numLabels, train, & splitTrainTest returned by the generator. */
+    public LFWDataSetIterator(int batchSize, int numExamples, int numLabels, boolean train, double splitTrainTest) {
+        this(batchSize, numExamples, new int[] {LFWLoader.HEIGHT, LFWLoader.WIDTH, LFWLoader.CHANNELS}, numLabels,
+                        false, null, train, splitTrainTest, null, new Random(System.currentTimeMillis()));
+    }
+
+    /** Loads images with given  batchSize, numExamples, imgDim, numLabels, useSubset, train, splitTrainTest & Random returned by the generator. */
+    public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset,
+                    boolean train, double splitTrainTest, Random rng) {
+        this(batchSize, numExamples, imgDim, numLabels, useSubset, LFWLoader.LABEL_PATTERN, train, splitTrainTest, null,
+                        rng);
+    }
+
+    /** Loads images with given  batchSize, numExamples, imgDim, numLabels, useSubset, train, splitTrainTest & Random returned by the generator. */
+    public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset,
+                    PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
+        this(batchSize, numExamples, imgDim, numLabels, useSubset, labelGenerator, train, splitTrainTest, null, rng);
+    }
+
+    /**
+     * Create LFW data specific iterator
+     * @param batchSize the batch size of the examples
+     * @param numExamples the overall number of examples
+     * @param imgDim an array of height, width and channels
+     * @param numLabels the overall number of examples
+     * @param useSubset use a subset of the LFWDataSet
+     * @param labelGenerator path label generator to use
+     * @param train true if use train value
+     * @param splitTrainTest the percentage to split data for train and remainder goes to test
+     * @param imageTransform how to transform the image
+    
+     * @param rng random number to lock in batch shuffling
+     * */
+    public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset,
+                    PathLabelGenerator labelGenerator, boolean train, double splitTrainTest,
+                    ImageTransform imageTransform, Random rng) {
+        super(new LFWLoader(imgDim, imageTransform, useSubset).getRecordReader(batchSize, numExamples, imgDim,
+                        numLabels, labelGenerator, train, splitTrainTest, rng), batchSize, 1, numLabels);
+    }
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/MnistDataSetIterator.java

@@ -0,0 +1,89 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.impl;
+
+import org.deeplearning4j.datasets.fetchers.MnistDataFetcher;
+import org.nd4j.linalg.dataset.api.iterator.BaseDatasetIterator;
+
+import java.io.File;
+import java.io.IOException;
+
+public class MnistDataSetIterator extends BaseDatasetIterator {
+
+    public MnistDataSetIterator(int batch, int numExamples) throws IOException {
+        this(batch, numExamples, false);
+    }
+
+    /**Get the specified number of examples for the MNIST training data set.
+     * @param batch the batch size of the examples
+     * @param numExamples the overall number of examples
+     * @param binarize whether to binarize mnist or not
+     * @throws IOException
+     */
+    public MnistDataSetIterator(int batch, int numExamples, boolean binarize) throws IOException {
+        this(batch, numExamples, binarize, true, false, 0);
+    }
+
+    /** Constructor to get the full MNIST data set (either test or train sets) without binarization (i.e., just normalization
+     * into range of 0 to 1), with shuffling based on a random seed.
+     * @param batchSize
+     * @param train
+     * @throws IOException
+     */
+    public MnistDataSetIterator(int batchSize, boolean train, int seed) throws IOException {
+        this(batchSize, (train ? MnistDataFetcher.NUM_EXAMPLES : MnistDataFetcher.NUM_EXAMPLES_TEST), false, train,
+                true, seed);
+    }
+
+    /**Get the specified number of MNIST examples (test or train set), with optional shuffling and binarization.
+     * @param batch Size of each patch
+     * @param numExamples total number of examples to load
+     * @param binarize whether to binarize the data or not (if false: normalize in range 0 to 1)
+     * @param train Train vs. test set
+     * @param shuffle whether to shuffle the examples
+     * @param rngSeed random number generator seed to use when shuffling examples
+     */
+    public MnistDataSetIterator(int batch, int numExamples, boolean binarize, boolean train, boolean shuffle,
+                                long rngSeed) throws IOException {
+        this(batch, numExamples, binarize,train,shuffle,rngSeed,null);
+    }
+
+
+    /**Get the specified number of MNIST examples (test or train set), with optional shuffling and binarization.
+     * @param batch Size of each patch
+     * @param numExamples total number of examples to load
+     * @param binarize whether to binarize the data or not (if false: normalize in range 0 to 1)
+     * @param train Train vs. test set
+     * @param shuffle whether to shuffle the examples
+     * @param rngSeed random number generator seed to use when shuffling examples
+     */
+    public MnistDataSetIterator(int batch, int numExamples, boolean binarize, boolean train, boolean shuffle,
+                                long rngSeed, File topLevelDir) throws IOException {
+        super(batch, numExamples, new MnistDataFetcher(binarize, train, shuffle, rngSeed, numExamples,topLevelDir));
+    }
+
+
+    public void close() {
+        MnistDataFetcher mnistDataFetcher = (MnistDataFetcher) fetcher;
+        mnistDataFetcher.close();
+    }
+
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/TinyImageNetDataSetIterator.java

@@ -0,0 +1,127 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.impl;
+
+import lombok.Getter;
+import org.apache.commons.io.FileUtils;
+import org.datavec.image.transform.ImageTransform;
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator;
+import org.deeplearning4j.datasets.fetchers.DataSetType;
+import org.deeplearning4j.datasets.fetchers.TinyImageNetFetcher;
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class TinyImageNetDataSetIterator extends RecordReaderDataSetIterator {
+
+    @Getter
+    protected DataSetPreProcessor preProcessor;
+
+    /**
+     * Create an iterator for the training set, with random iteration order (RNG seed fixed to 123)
+     *
+     * @param batchSize Minibatch size for the iterator
+     */
+    public TinyImageNetDataSetIterator(int batchSize) {
+        this(batchSize, null, DataSetType.TRAIN, null, 123);
+    }
+
+    /**
+     * * Create an iterator for the training or test set, with random iteration order (RNG seed fixed to 123)
+     *
+     * @param batchSize Minibatch size for the iterator
+     * @param set       The dataset (train or test)
+     */
+    public TinyImageNetDataSetIterator(int batchSize, DataSetType set) {
+        this(batchSize, null, set, null, 123);
+    }
+
+    /**
+     * Get the Tiny ImageNet iterator with specified train/test set, with random iteration order (RNG seed fixed to 123)
+     *
+     * @param batchSize Size of each patch
+     * @param imgDim Dimensions of desired output - for example, {64, 64}
+     * @param set Train, test, or validation
+     */
+    public TinyImageNetDataSetIterator(int batchSize, int[] imgDim, DataSetType set) {
+        this(batchSize, imgDim, set, null, 123);
+    }
+
+    /**
+     * Get the Tiny ImageNet iterator with specified train/test set and (optional) custom transform.
+     *
+     * @param batchSize Size of each patch
+     * @param imgDim Dimensions of desired output - for example, {64, 64}
+     * @param set Train, test, or validation
+     * @param imageTransform Additional image transform for output
+     * @param rngSeed random number generator seed to use when shuffling examples
+     */
+    public TinyImageNetDataSetIterator(int batchSize, int[] imgDim, DataSetType set,
+                                       ImageTransform imageTransform, long rngSeed) {
+        super(new TinyImageNetFetcher().getRecordReader(rngSeed, imgDim, set, imageTransform), batchSize, 1, TinyImageNetFetcher.NUM_LABELS);
+    }
+
+    /**
+     * Get the labels - either in "categories" (imagenet synsets format, "n01910747" or similar) or human-readable format,
+     * such as "jellyfish"
+     * @param categories If true: return category/synset format; false: return "human readable" label format
+     * @return Labels
+     */
+    public static List<String> getLabels(boolean categories){
+        List<String> rawLabels = new TinyImageNetDataSetIterator(1).getLabels();
+        if(categories){
+            return rawLabels;
+        }
+
+        //Otherwise, convert to human-readable format, using 'words.txt' file
+        File baseDir = DL4JResources.getDirectory(ResourceType.DATASET, TinyImageNetFetcher.LOCAL_CACHE_NAME);
+        File labelFile = new File(baseDir, TinyImageNetFetcher.WORDS_FILENAME);
+        List<String> lines;
+        try {
+            lines = FileUtils.readLines(labelFile, StandardCharsets.UTF_8);
+        } catch (IOException e){
+            throw new RuntimeException("Error reading label file", e);
+        }
+
+        Map<String,String> map = new HashMap<>();
+        for(String line : lines){
+            String[] split = line.split("\t");
+            map.put(split[0], split[1]);
+        }
+
+        List<String> outLabels = new ArrayList<>(rawLabels.size());
+        for(String s : rawLabels){
+            String s2 = map.get(s);
+            Preconditions.checkState(s2 != null, "Label \"%s\" not found in labels.txt file");
+            outLabels.add(s2);
+        }
+        return outLabels;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/UciSequenceDataSetIterator.java

@@ -0,0 +1,62 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.iterator.impl;
+
+import org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator;
+import org.deeplearning4j.datasets.fetchers.DataSetType;
+import org.deeplearning4j.datasets.fetchers.UciSequenceDataFetcher;
+import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
+
+public class UciSequenceDataSetIterator extends SequenceRecordReaderDataSetIterator {
+
+    protected DataSetPreProcessor preProcessor;
+
+    /**
+     * Create an iterator for the training set, with the specified minibatch size. Randomized with RNG seed 123
+     *
+     * @param batchSize Minibatch size
+     */
+    public UciSequenceDataSetIterator(int batchSize) {
+        this(batchSize, DataSetType.TRAIN, 123);
+    }
+
+    /**
+     * Create an iterator for the training or test set, with the specified minibatch size. Randomized with RNG seed 123
+     *
+     * @param batchSize Minibatch size
+     * @param set       Set: training or test
+     */
+    public UciSequenceDataSetIterator(int batchSize, DataSetType set) {
+        this(batchSize, set, 123);
+    }
+
+    /**
+     * Create an iterator for the training or test set, with the specified minibatch size
+     *
+     * @param batchSize Minibatch size
+     * @param set       Set: training or test
+     * @param rngSeed   Random number generator seed to use for randomization
+     */
+    public UciSequenceDataSetIterator(int batchSize, DataSetType set, long rngSeed) {
+        super(new UciSequenceDataFetcher().getRecordReader(rngSeed, set), batchSize, UciSequenceDataFetcher.NUM_LABELS, 1);
+        // last parameter is index of label
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistDbFile.java

@@ -0,0 +1,125 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.mnist;
+
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+public abstract class MnistDbFile extends RandomAccessFile {
+    private int count;
+
+
+    /**
+     * Creates new instance and reads the header information.
+     * 
+     * @param name
+     *            the system-dependent filename
+     * @param mode
+     *            the access mode
+     * @throws IOException
+     * @throws FileNotFoundException
+     * @see RandomAccessFile
+     */
+    public MnistDbFile(String name, String mode) throws IOException {
+        super(name, mode);
+        if (getMagicNumber() != readInt()) {
+            throw new RuntimeException(
+                            "This MNIST DB file " + name + " should start with the number " + getMagicNumber() + ".");
+        }
+        count = readInt();
+    }
+
+    /**
+     * MNIST DB files start with unique integer number.
+     * 
+     * @return integer number that should be found in the beginning of the file.
+     */
+    protected abstract int getMagicNumber();
+
+    /**
+     * The current entry index.
+     * 
+     * @return long
+     * @throws IOException
+     */
+    public long getCurrentIndex() throws IOException {
+        return (getFilePointer() - getHeaderSize()) / getEntryLength() + 1;
+    }
+
+    /**
+     * Set the required current entry index.
+     * 
+     * @param curr
+     *            the entry index
+     */
+    public void setCurrentIndex(long curr) {
+        try {
+            if (curr < 0 || curr > count) {
+                throw new RuntimeException(curr + " is not in the range 0 to " + count);
+            }
+            seek(getHeaderSize() + curr * getEntryLength());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public int getHeaderSize() {
+        return 8; // two integers
+    }
+
+    /**
+     * Number of bytes for each entry.
+     * Defaults to 1.
+     * 
+     * @return int
+     */
+    public int getEntryLength() {
+        return 1;
+    }
+
+    /**
+     * Move to the next entry.
+     * 
+     * @throws IOException
+     */
+    public void next() throws IOException {
+        if (getCurrentIndex() < count) {
+            skipBytes(getEntryLength());
+        }
+    }
+
+    /**
+     * Move to the previous entry.
+     * 
+     * @throws IOException
+     */
+    public void prev() throws IOException {
+        if (getCurrentIndex() > 0) {
+            seek(getFilePointer() - getEntryLength());
+        }
+    }
+
+    public int getCount() {
+        return count;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistImageFile.java

@@ -0,0 +1,130 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.mnist;
+
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+
+public class MnistImageFile extends MnistDbFile {
+    private int rows;
+    private int cols;
+
+    /**
+     * Creates new MNIST database image file ready for reading.
+     * 
+     * @param name
+     *            the system-dependent filename
+     * @param mode
+     *            the access mode
+     * @throws IOException
+     * @throws FileNotFoundException
+     */
+    public MnistImageFile(String name, String mode) throws IOException {
+        super(name, mode);
+
+        // read header information
+        rows = readInt();
+        cols = readInt();
+    }
+
+    /**
+     * Reads the image at the current position.
+     * 
+     * @return matrix representing the image
+     * @throws IOException
+     */
+    public int[][] readImage() throws IOException {
+        int[][] dat = new int[getRows()][getCols()];
+        for (int i = 0; i < getCols(); i++) {
+            for (int j = 0; j < getRows(); j++) {
+                dat[i][j] = readUnsignedByte();
+            }
+        }
+        return dat;
+    }
+
+    /** Read the specified number of images from the current position, to a byte[nImages][rows*cols]
+     * Note that MNIST data set is stored as unsigned bytes; this method returns signed bytes without conversion
+     * (i.e., same bits, but requires conversion before use)
+     * @param nImages Number of images
+     */
+    public byte[][] readImagesUnsafe(int nImages) throws IOException {
+        byte[][] out = new byte[nImages][0];
+        for (int i = 0; i < nImages; i++) {
+            out[i] = new byte[rows * cols];
+            read(out[i]);
+        }
+        return out;
+    }
+
+    /**
+     * Move the cursor to the next image.
+     * 
+     * @throws IOException
+     */
+    public void nextImage() throws IOException {
+        super.next();
+    }
+
+    /**
+     * Move the cursor to the previous image.
+     * 
+     * @throws IOException
+     */
+    public void prevImage() throws IOException {
+        super.prev();
+    }
+
+    @Override
+    protected int getMagicNumber() {
+        return 2051;
+    }
+
+    /**
+     * Number of rows per image.
+     * 
+     * @return int
+     */
+    public int getRows() {
+        return rows;
+    }
+
+    /**
+     * Number of columns per image.
+     * 
+     * @return int
+     */
+    public int getCols() {
+        return cols;
+    }
+
+    @Override
+    public int getEntryLength() {
+        return cols * rows;
+    }
+
+    @Override
+    public int getHeaderSize() {
+        return super.getHeaderSize() + 8; // to more integers - rows and columns
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistLabelFile.java

@@ -0,0 +1,71 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.mnist;
+
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+
+/**
+ * 
+ * MNIST database label file.
+ * 
+ */
+public class MnistLabelFile extends MnistDbFile {
+
+    /**
+     * Creates new MNIST database label file ready for reading.
+     * 
+     * @param name
+     *            the system-dependent filename
+     * @param mode
+     *            the access mode
+     * @throws IOException
+     * @throws FileNotFoundException
+     */
+    public MnistLabelFile(String name, String mode) throws IOException {
+        super(name, mode);
+    }
+
+    /**
+     * Reads the integer at the current position.
+     * 
+     * @return integer representing the label
+     * @throws IOException
+     */
+    public int readLabel() throws IOException {
+        return readUnsignedByte();
+    }
+
+    /** Read the specified number of labels from the current position*/
+    public int[] readLabels(int num) throws IOException {
+        int[] out = new int[num];
+        for (int i = 0; i < num; i++)
+            out[i] = readLabel();
+        return out;
+    }
+
+    @Override
+    protected int getMagicNumber() {
+        return 2049;
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistManager.java

@@ -0,0 +1,188 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.datasets.mnist;
+
+
+import lombok.SneakyThrows;
+import org.deeplearning4j.datasets.fetchers.MnistDataFetcher;
+import org.nd4j.common.base.Preconditions;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+
+
+public class MnistManager {
+    MnistImageFile images;
+    private MnistLabelFile labels;
+
+    private byte[][] imagesArr;
+    private int[] labelsArr;
+    private static final int HEADER_SIZE = 8;
+
+    /**
+     * Writes the given image in the given file using the PPM data format.
+     *
+     * @param image
+     * @param ppmFileName
+     * @throws IOException
+     */
+    public static void writeImageToPpm(int[][] image, String ppmFileName) throws IOException {
+        try (BufferedWriter ppmOut = new BufferedWriter(new FileWriter(ppmFileName))) {
+            int rows = image.length;
+            int cols = image[0].length;
+            ppmOut.write("P3\n");
+            ppmOut.write("" + rows + " " + cols + " 255\n");
+            for (int i = 0; i < rows; i++) {
+                StringBuilder s = new StringBuilder();
+                for (int j = 0; j < cols; j++) {
+                    s.append(image[i][j] + " " + image[i][j] + " " + image[i][j] + "  ");
+                }
+                ppmOut.write(s.toString());
+            }
+        }
+
+    }
+
+    @SneakyThrows
+    public long getCurrent() {
+        return labels.getCurrentIndex();
+    }
+
+
+
+    /**
+     * Constructs an instance managing the two given data files. Supports
+     * <code>NULL</code> value for one of the arguments in case reading only one
+     * of the files (images and labels) is required.
+     *
+     * @param imagesFile
+     *            Can be <code>NULL</code>. In that case all future operations
+     *            using that file will fail.
+     * @param labelsFile
+     *            Can be <code>NULL</code>. In that case all future operations
+     *            using that file will fail.
+     * @throws IOException
+     */
+    public MnistManager(String imagesFile, String labelsFile, boolean train) throws IOException {
+        this(imagesFile, labelsFile, train ? MnistDataFetcher.NUM_EXAMPLES : MnistDataFetcher.NUM_EXAMPLES_TEST);
+    }
+
+
+
+    public MnistManager(String imagesFile, String labelsFile, int numExamples) throws IOException {
+        if (imagesFile != null) {
+            images = new MnistImageFile(imagesFile, "r");
+            imagesArr = images.readImagesUnsafe(numExamples);
+        }
+        if (labelsFile != null) {
+            labels = new MnistLabelFile(labelsFile, "r");
+            labelsArr = labels.readLabels(numExamples);
+        }
+    }
+
+    public MnistManager(String imagesFile, String labelsFile) throws IOException {
+        this(imagesFile, labelsFile, true);
+    }
+
+    /**
+     * Reads the current image.
+     *
+     * @return matrix
+     * @throws IOException
+     */
+    public int[][] readImage() throws IOException {
+        if (images == null) {
+            throw new IllegalStateException("Images file not initialized.");
+        }
+        return images.readImage();
+    }
+
+    public byte[] readImageUnsafe(int i) {
+        Preconditions.checkArgument(i < imagesArr.length);
+        return imagesArr[i];
+    }
+
+    /**
+     * Set the position to be read.
+     *
+     * @param index
+     */
+    public void setCurrent(int index) {
+        images.setCurrentIndex(index);
+        labels.setCurrentIndex(index);
+    }
+
+    /**
+     * Reads the current label.
+     *
+     * @return int
+     * @throws IOException
+     */
+    public int readLabel() throws IOException {
+        if (labels == null) {
+            throw new IllegalStateException("labels file not initialized.");
+        }
+        return labels.readLabel();
+    }
+
+    public int readLabel(int i) {
+        return labelsArr[i];
+    }
+
+    /**
+     * Get the underlying images file as {@link MnistImageFile}.
+     *
+     * @return {@link MnistImageFile}.
+     */
+    public MnistImageFile getImages() {
+        return images;
+    }
+
+    /**
+     * Get the underlying labels file as {@link MnistLabelFile}.
+     *
+     * @return {@link MnistLabelFile}.
+     */
+    public MnistLabelFile getLabels() {
+        return labels;
+    }
+
+    /**
+     * Close any resources opened by the manager.
+     */
+    public void close() {
+        if (images != null) {
+            try {
+                images.close();
+            } catch (IOException e) {
+            }
+            images = null;
+        }
+        if (labels != null) {
+            try {
+                labels.close();
+            } catch (IOException e) {
+            }
+            labels = null;
+        }
+    }
+}

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java9/module-info.java

@@ -0,0 +1,15 @@
+open module deeplearning4j.datasets {
+    requires commons.io;
+    requires lombok;
+    requires nd4j.common;
+    requires slf4j.api;
+    requires datavec.api;
+    requires datavec.data.image;
+    requires deeplearning4j.datavec.iterators;
+    requires nd4j.api;
+    requires resources;
+    exports org.deeplearning4j.datasets.base;
+    exports org.deeplearning4j.datasets.fetchers;
+    exports org.deeplearning4j.datasets.iterator.impl;
+    exports org.deeplearning4j.datasets.mnist;
+}