chore: import upstream snapshot with attribution

resources/src/main/java/org/deeplearning4j/common/resources/DL4JResources.java

@@ -0,0 +1,150 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.common.resources;
+
+import lombok.NonNull;
+import org.deeplearning4j.config.DL4JSystemProperties;
+import org.nd4j.common.base.Preconditions;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+public class DL4JResources {
+
+    /**
+     * @deprecated Use {@link DL4JSystemProperties#DL4J_RESOURCES_DIR_PROPERTY}
+     */
+    @Deprecated
+    public static final String DL4J_RESOURCES_DIR_PROPERTY = DL4JSystemProperties.DL4J_RESOURCES_DIR_PROPERTY;
+    /**
+     * @deprecated Use {@link DL4JSystemProperties#DL4J_RESOURCES_BASE_URL_PROPERTY}
+     */
+    @Deprecated
+    public static final String DL4J_BASE_URL_PROPERTY = DL4JSystemProperties.DL4J_RESOURCES_BASE_URL_PROPERTY;
+    private static final String DL4J_DEFAULT_URL = "https://dl4jdata.blob.core.windows.net/";
+
+    private static File baseDirectory;
+    private static String baseURL;
+
+    static {
+        resetBaseDirectoryLocation();
+
+        String property = System.getProperty(DL4JSystemProperties.DL4J_RESOURCES_BASE_URL_PROPERTY);
+        if(property != null){
+            baseURL = property;
+        } else {
+            baseURL = DL4J_DEFAULT_URL;
+        }
+
+    }
+
+    /**
+     * Set the base download URL for (most) DL4J datasets and models.<br>
+     * This usually doesn't need to be set manually unless there is some issue with the default location
+     * @param baseDownloadURL Base download URL to set. For example, https://dl4jdata.blob.core.windows.net/
+     */
+    public static void setBaseDownloadURL(@NonNull String baseDownloadURL){
+        baseURL = baseDownloadURL;
+    }
+
+    /**
+     * @return The base URL hosting DL4J datasets and models
+     */
+    public static String getBaseDownloadURL(){
+        return baseURL;
+    }
+
+    /**
+     * Get the URL relative to the base URL.<br>
+     * For example, if baseURL is "https://dl4jdata.blob.core.windows.net/", and relativeToBase is "/datasets/iris.dat"
+     * this simply returns "https://dl4jdata.blob.core.windows.net/datasets/iris.dat"
+     *
+     * @param relativeToBase Relative URL
+     * @return URL
+     * @throws MalformedURLException For bad URL
+     */
+    public static URL getURL(String relativeToBase) throws MalformedURLException {
+        return new URL(getURLString(relativeToBase));
+    }
+
+    /**
+     * Get the URL relative to the base URL as a String.<br>
+     * For example, if baseURL is "https://dl4jdata.blob.core.windows.net/", and relativeToBase is "/datasets/iris.dat"
+     * this simply returns "https://dl4jdata.blob.core.windows.net/datasets/iris.dat"
+     *
+     * @param relativeToBase Relative URL
+     * @return URL
+     * @throws MalformedURLException For bad URL
+     */
+    public static String getURLString(String relativeToBase) {
+        if(relativeToBase.startsWith("/")){
+            relativeToBase = relativeToBase.substring(1);
+        }
+        return baseURL + relativeToBase;
+    }
+
+    /**
+     * Reset to the default directory, or the directory set via the {@link DL4JSystemProperties#DL4J_RESOURCES_DIR_PROPERTY} system property,
+     * org.deeplearning4j.resources.directory
+     */
+    public static void resetBaseDirectoryLocation(){
+        String property = System.getProperty(DL4JSystemProperties.DL4J_RESOURCES_DIR_PROPERTY);
+        if(property != null){
+            baseDirectory = new File(property);
+        } else {
+            baseDirectory = new File(System.getProperty("user.home"), ".deeplearning4j");
+        }
+
+        if(!baseDirectory.exists()){
+            baseDirectory.mkdirs();
+        }
+    }
+
+    /**
+     * Set the base directory for local storage of files. Default is: {@code new File(System.getProperty("user.home"), ".deeplearning4j")}
+     * @param f Base directory to use for resources
+     */
+    public static void setBaseDirectory(@NonNull File f){
+        Preconditions.checkState(f.exists() && f.isDirectory(), "Specified base directory does not exist and/or is not a directory: %s", f.getAbsolutePath());
+        baseDirectory = f;
+    }
+
+    /**
+     * @return The base storage directory for DL4J resources
+     */
+    public static File getBaseDirectory(){
+        return baseDirectory;
+    }
+
+    /**
+     * Get the storage location for the specified resource type and resource name
+     * @param resourceType Type of resource
+     * @param resourceName Name of the resource
+     * @return The root directory. Creates the directory and any parent directories, if required
+     */
+    public static File getDirectory(ResourceType resourceType, String resourceName){
+        File f = new File(baseDirectory, resourceType.resourceName());
+        f = new File(f, resourceName);
+        f.mkdirs();
+        return f;
+    }
+}

resources/src/main/java/org/deeplearning4j/common/resources/ResourceType.java

@@ -0,0 +1,42 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.common.resources;
+
+public enum ResourceType {
+
+    DATASET,
+    ZOO_MODEL,
+    RESOURCE;
+
+    public String resourceName(){
+        switch (this){
+            case DATASET:
+                return "data";
+            case ZOO_MODEL:
+                return "models";
+            case RESOURCE:
+                return "resources";
+            default:
+                return this.toString();
+        }
+    }
+
+}

resources/src/main/java/org/deeplearning4j/common/util/ND4JFileUtils.java

@@ -0,0 +1,52 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.common.util;
+
+import org.deeplearning4j.config.DL4JSystemProperties;
+
+import java.io.File;
+import java.io.IOException;
+
+public class ND4JFileUtils {
+
+    private ND4JFileUtils(){ }
+
+    /**
+     * Create a temporary file in the location specified by {@link DL4JSystemProperties#DL4J_TEMP_DIR_PROPERTY} if set,
+     * or the default temporary directory (usually specified by java.io.tmpdir system property)
+     * @param prefix Prefix for generating file's name; must be at least 3 characeters
+     * @param suffix Suffix for generating file's name; may be null (".tmp" will be used if null)
+     * @return A temporary file
+     */
+    public static File createTempFile(String prefix, String suffix) {
+        String p = System.getProperty(DL4JSystemProperties.DL4J_TEMP_DIR_PROPERTY);
+        try {
+            if (p == null || p.isEmpty()) {
+                return File.createTempFile(prefix, suffix);
+            } else {
+                return File.createTempFile(prefix, suffix, new File(p));
+            }
+        } catch (IOException e){
+            throw new RuntimeException("Error creating temporary file", e);
+        }
+    }
+
+}

resources/src/main/java/org/deeplearning4j/config/DL4JClassLoading.java

@@ -0,0 +1,124 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.config;
+
+import lombok.extern.slf4j.Slf4j;
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.common.config.ND4JClassLoading;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.Objects;
+import java.util.ServiceLoader;
+
+@Slf4j
+public class DL4JClassLoading {
+    private static ClassLoader dl4jClassloader = ND4JClassLoading.getNd4jClassloader();
+
+    private DL4JClassLoading() {
+    }
+
+    public static ClassLoader getDl4jClassloader() {
+        return DL4JClassLoading.dl4jClassloader;
+    }
+
+    public static void setDl4jClassloaderFromClass(Class<?> clazz) {
+        setDl4jClassloader(clazz.getClassLoader());
+    }
+
+    public static void setDl4jClassloader(ClassLoader dl4jClassloader) {
+        DL4JClassLoading.dl4jClassloader = dl4jClassloader;
+        log.debug("Global class-loader for DL4J was changed.");
+    }
+
+    public static boolean classPresentOnClasspath(String className) {
+        return classPresentOnClasspath(className, dl4jClassloader);
+    }
+
+    public static boolean classPresentOnClasspath(String className, ClassLoader classLoader) {
+        return loadClassByName(className, false, classLoader) != null;
+    }
+
+    public static <T> Class<T> loadClassByName(String className) {
+        return loadClassByName(className, true, dl4jClassloader);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static <T> Class<T> loadClassByName(String className, boolean initialize, ClassLoader classLoader) {
+        try {
+            return (Class<T>) Class.forName(className, initialize, classLoader);
+        } catch (ClassNotFoundException classNotFoundException) {
+            log.error(String.format("Cannot find class [%s] of provided class-loader.", className));
+            return null;
+        }
+    }
+
+    public static <T> T createNewInstance(String className) {
+        return createNewInstance(className, Object.class, new Object[0]);//or null;
+    }
+    
+    public static <T> T createNewInstance(String className, Object[] args) {
+       
+    	return createNewInstance(className, Object.class, args);
+    }
+
+    public static <T> T createNewInstance(String className, Class<? super T> superclass) {
+        return createNewInstance(className, superclass, new Class<?>[]{}, new Object[]{});
+    }
+
+    public static <T> T createNewInstance(String className, Class<? super T> superclass, Object[] args) {
+        Class<?>[] parameterTypes = new Class<?>[args.length];
+        for (int i = 0; i < args.length; i++) {
+            Object arg = args[i];
+            Objects.requireNonNull(arg);
+            parameterTypes[i] = arg.getClass();
+        }
+
+        return createNewInstance(className, superclass, parameterTypes, args);
+    }
+
+    public static <T> T createNewInstance(
+            String className,
+            Class<? super T> superclass,
+            Class<?>[] parameterTypes,
+            Object [] args) {
+        try {
+            Class<Object> loadedClass =  DL4JClassLoading
+                    .loadClassByName(className);
+            Preconditions.checkNotNull(loadedClass,"Attempted to load class " + className + " but failed. No class found with this name.");
+            return (T) loadedClass
+                    .asSubclass(superclass)
+                    .getDeclaredConstructor(parameterTypes)
+                    .newInstance(args);
+        } catch (InstantiationException | IllegalAccessException | InvocationTargetException
+                | NoSuchMethodException instantiationException) {
+            log.error(String.format("Cannot create instance of class '%s'.", className), instantiationException);
+            throw new RuntimeException(instantiationException);
+        }
+    }
+
+    public static <S> ServiceLoader<S> loadService(Class<S> serviceClass) {
+        return loadService(serviceClass, dl4jClassloader);
+    }
+
+    public static <S> ServiceLoader<S> loadService(Class<S> serviceClass, ClassLoader classLoader) {
+        return ServiceLoader.load(serviceClass, classLoader);
+    }
+}

resources/src/main/java/org/deeplearning4j/config/DL4JEnvironmentVars.java

@@ -0,0 +1,35 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.config;
+
+public class DL4JEnvironmentVars {
+
+    private DL4JEnvironmentVars(){ }
+
+
+    /**
+     * Applicability: Module dl4j-spark-parameterserver_2.xx<br>
+     * Usage: A fallback for determining the local IP for a Spark training worker, if other approaches
+     * fail to determine the local IP
+     */
+    public static final String DL4J_VOID_IP = "DL4J_VOID_IP";
+
+}

resources/src/main/java/org/deeplearning4j/config/DL4JSystemProperties.java

@@ -0,0 +1,130 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.config;
+
+public class DL4JSystemProperties {
+
+    private DL4JSystemProperties(){ }
+
+    /**
+     * Applicability: DL4J ModelSerializer, ModelGuesser, Keras model import<br>
+     * Description: Specify the local directory where temporary files will be written. If not specified, the default
+     * Java temporary directory (java.io.tmpdir system property) will generally be used.
+     */
+    public static final String DL4J_TEMP_DIR_PROPERTY = "org.deeplearning4j.tempdir";
+
+    /**
+     * Applicability: Numerous modules, including deeplearning4j-datasets and deeplearning4j-zoo<br>
+     * Description: Used to set the local location for downloaded remote resources such as datasets (like MNIST) and
+     * pretrained models in the model zoo. Default value is set via {@code new File(System.getProperty("user.home"), ".deeplearning4j")}.
+     * Setting this can be useful if the system drive has limited space/performance, a shared location for all users
+     * should be used instead, or if user.home isn't set for some reason.
+     */
+    public static final String DL4J_RESOURCES_DIR_PROPERTY = "org.deeplearning4j.resources.directory";
+
+    /**
+     * Applicability: Numerous modules, including deeplearning4j-datasets and deeplearning4j-zoo<br>
+     * Description: Used to set the base URL for hosting of resources such as datasets (like MNIST) and pretrained
+     * models in the model zoo. This is provided as a fallback in case the location of these files changes; it
+     * also allows for (in principle) a local mirror of these files.<br>
+     * NOTE: Changing this to a location without the same files and file structure as the DL4J resource hosting is likely
+     * to break external resource dowloading in DL4J!
+     */
+    public static final String DL4J_RESOURCES_BASE_URL_PROPERTY = "org.deeplearning4j.resources.baseurl";
+
+    /**
+     * Applicability: deeplearning4j-nn<br>
+     * Description: DL4J writes some crash dumps to disk when an OOM exception occurs - this functionality is enabled
+     * by default. This is to help users identify the cause of the OOM - i.e., where native memory is actually consumed.
+     * This system property can be used to disable memory crash reporting.
+     * @see #CRASH_DUMP_OUTPUT_DIRECTORY_PROPERTY For configuring the output directory
+     */
+    public static final String CRASH_DUMP_ENABLED_PROPERTY = "org.deeplearning4j.crash.reporting.enabled";
+
+    /**
+     * Applicability: deeplearning4j-nn<br>
+     * Description: DL4J writes some crash dumps to disk when an OOM exception occurs - this functionality is enabled
+     * by default. This system property can be use to customize the output directory for memory crash reporting. By default,
+     * the current working directory ({@code  System.getProperty("user.dir")} or {@code new File("")}) will be used
+     * @see #CRASH_DUMP_ENABLED_PROPERTY To disable crash dump reporting
+     */
+    public static final String CRASH_DUMP_OUTPUT_DIRECTORY_PROPERTY = "org.deeplearning4j.crash.reporting.directory";
+
+    /**
+     * Applicability: deeplearning4j-ui<br>
+     * Description: The DL4J training UI (StatsListener + UIServer.getInstance().attach(ss)) will subsample the number
+     * of chart points when a lot of data is present - i.e., only a maximum number of points will be shown on each chart.
+     * This is to reduce the UI bandwidth requirements and client-side rendering cost.
+     * To increase the number of points in charts, set this property to a larger value. Default: 512 values
+     */
+    public static final String CHART_MAX_POINTS_PROPERTY = "org.deeplearning4j.ui.maxChartPoints";
+
+
+    /**
+     * Applicability: deeplearning4j-vertx (deeplearning4j-ui)<br>
+     * Description: This property sets the port that the UI will be available on. Default port: 9000.
+     * Set to 0 for a random port.
+     */
+    public static final String UI_SERVER_PORT_PROPERTY = "org.deeplearning4j.ui.port";
+
+    /**
+     * Applicability: dl4j-spark_2.xx - NTPTimeSource class (mainly used in ParameterAveragingTrainingMaster when stats
+     * collection is enabled; not enabled by default)<br>
+     * Description: This sets the NTP (network time protocol) server to be used when collecting stats. Default: 0.pool.ntp.org
+     */
+    public static final String NTP_SOURCE_SERVER_PROPERTY = "org.deeplearning4j.spark.time.NTPTimeSource.server";
+
+    /**
+     * Applicability: dl4j-spark_2.xx - NTPTimeSource class (mainly used in ParameterAveragingTrainingMaster when stats
+     * collection is enabled; not enabled by default)<br>
+     * Description: This sets the NTP (network time protocol) update frequency in milliseconds. Default: 1800000 (30 minutes)
+     */
+    public static final String NTP_SOURCE_UPDATE_FREQUENCY_MS_PROPERTY = "org.deeplearning4j.spark.time.NTPTimeSource.frequencyms";
+
+    /**
+     * Applicability: dl4j-spark_2.xx - mainly used in ParameterAveragingTrainingMaster when stats collection is enabled;
+     * not enabled by default<br>
+     * Description: This sets the time source to use for spark stats. Default: {@code org.deeplearning4j.spark.time.NTPTimeSource}
+     */
+    public static final String TIMESOURCE_CLASSNAME_PROPERTY = "org.deeplearning4j.spark.time.TimeSource";
+
+
+    /**
+     * Applicability: {@code org.deeplearning4j.nn.layers.HelperUtils}
+     * Used in whether to disable the helpers or not.
+     */
+    public static final String DISABLE_HELPER_PROPERTY = "org.eclipse.deeplearning4j.helpers.disable";
+    public final static String HELPER_DISABLE_DEFAULT_VALUE = "true";
+
+    /**
+     * Applicability: NLP SkipGram and CBOW
+     * modules controlling the max cache size
+     * for on heap parameters such as learning rates and random values.
+     */
+    public static final String NLP_CACHE_SIZE = "org.eclipse.deeplearning4j.nlp.cachesize";
+
+    /**
+     * Applicability: NLP SkipGram and CBOW
+     * modules controlling the max queue size
+     * for each cache value.
+     */
+    public static final String NLP_QUEUE_SIZE = "org.eclipse.deeplearning4j.nlp.queuesize";
+}

resources/src/main/java/org/deeplearning4j/omnihub/OmnihubConfig.java

@@ -0,0 +1,56 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.deeplearning4j.omnihub;
+
+import java.io.File;
+
+public class OmnihubConfig {
+
+    public final static String OMNIHUB_HOME = "OMNIHUB_HOME";
+    public final static String OMNIHUB_URL = "OMNIHUB_URL";
+    public static final String DEFAULT_OMNIHUB_URL = "https://raw.githubusercontent.com/KonduitAI/omnihub-zoo/main";
+
+    /**
+     * Return the omnihub hurl defaulting to
+     * {@link #DEFAULT_OMNIHUB_URL}
+     * if the {@link #OMNIHUB_URL} is not specified.
+     * @return
+     */
+    public static String getOmnihubUrl() {
+        if(System.getenv(OMNIHUB_URL) != null) {
+            return System.getenv(OMNIHUB_URL);
+        } else {
+            return DEFAULT_OMNIHUB_URL;
+        }
+    }
+
+    /**
+     * return the default omnihub home at $USER/.omnihub or
+     * value of the environment variable {@link #OMNIHUB_HOME} if applicable
+     * @return
+     */
+    public static File getOmnihubHome() {
+        if(System.getenv(OMNIHUB_HOME) != null) {
+            return new File(OMNIHUB_HOME);
+        } else {
+            return new File(System.getProperty("user.home"),".omnihub");
+        }
+    }
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/BaseResource.java

@@ -0,0 +1,43 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.resources;
+
+public abstract class BaseResource implements DownloadableResource {
+    protected String fileName;
+    protected String archiveFileName;
+    protected String md5Sum;
+
+
+    public BaseResource(String fileName,String md5Sum,String archiveFileName) {
+        this.fileName = fileName;
+        this.archiveFileName = archiveFileName;
+        this.md5Sum = md5Sum;
+    }
+
+
+    public BaseResource(String fileName,String archiveFileName) {
+        this(fileName,"",archiveFileName);
+    }
+
+    @Override
+    public String archiveFileName() {
+        return archiveFileName;
+    }
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/CustomResource.java

@@ -0,0 +1,64 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.resources;
+
+import java.io.File;
+
+public class CustomResource extends BaseResource {
+
+    private String localCacheDirectory;
+    private String rootUrl;
+
+    public CustomResource(String fileName,String localCacheDirectory, String rootUrl) {
+        this(fileName,"",localCacheDirectory,rootUrl);
+    }
+
+    public CustomResource(String fileName, String archiveFile,String localCacheDirectory, String rootUrl) {
+        super(fileName,archiveFile);
+        this.localCacheDirectory = localCacheDirectory;
+        this.rootUrl = rootUrl;
+    }
+
+    public CustomResource(String fileName, String md5Sum, String archiveFileName, String localCacheDirectory, String rootUrl) {
+        super(fileName, md5Sum, archiveFileName);
+        this.localCacheDirectory = localCacheDirectory;
+        this.rootUrl = rootUrl;
+    }
+
+    @Override
+    public String fileName() {
+        return fileName;
+    }
+
+    @Override
+    public String rootUrl() {
+        return rootUrl;
+    }
+
+    @Override
+    public File localCacheDirectory() {
+        return new File(localCacheDirectory);
+    }
+
+    @Override
+    public ResourceType resourceType() {
+        return ResourceType.CUSTOM;
+    }
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/DataSetResource.java

@@ -0,0 +1,68 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.resources;
+
+
+import java.io.File;
+public class DataSetResource extends BaseResource {
+
+
+    private String localCacheDirectory;
+    private String rootUrl;
+
+
+    public DataSetResource(String fileName,String localCacheDirectory, String rootUrl) {
+        this(fileName,"",localCacheDirectory,rootUrl);
+    }
+
+    public DataSetResource(String fileName, String archiveFileName,String localCacheDirectory, String rootUrl) {
+        super(fileName,archiveFileName);
+        this.localCacheDirectory = localCacheDirectory;
+        this.rootUrl = rootUrl;
+    }
+
+    public DataSetResource(String fileName, String md5Sum, String archiveFileName, String localCacheDirectory, String rootUrl) {
+        super(fileName, md5Sum, archiveFileName);
+        this.localCacheDirectory = localCacheDirectory;
+        this.rootUrl = rootUrl;
+    }
+
+    @Override
+    public String fileName() {
+        return fileName;
+    }
+
+    @Override
+    public String rootUrl() {
+        return rootUrl;
+    }
+
+    @Override
+    public File localCacheDirectory() {
+        return new File(localCacheDirectory);
+    }
+
+    @Override
+    public ResourceType resourceType() {
+        return ResourceType.DATASET;
+    }
+
+
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/Dl4jZooResource.java

@@ -0,0 +1,63 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.resources;
+
+import org.deeplearning4j.common.resources.DL4JResources;
+
+import java.io.File;
+
+public class Dl4jZooResource extends BaseResource {
+
+    private String modelName;
+
+
+    public Dl4jZooResource(String fileName,String archiveName,String modelName) {
+        super(fileName,archiveName);
+        this.modelName = modelName;
+    }
+
+    public Dl4jZooResource(String fileName,String modelName) {
+        this(fileName,"",modelName);
+    }
+
+
+    @Override
+    public String fileName() {
+        return fileName;
+    }
+
+    @Override
+    public String rootUrl() {
+        return DL4JResources.getBaseDownloadURL() + "/models";
+    }
+
+    @Override
+    public File localCacheDirectory() {
+        File rootCacheDir = DL4JResources.getDirectory(org.deeplearning4j.common.resources.ResourceType.ZOO_MODEL, modelName);
+        return rootCacheDir;
+    }
+
+    @Override
+    public ResourceType resourceType() {
+        return ResourceType.LEGACY_ZOO;
+    }
+
+
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/DownloadResources.java

@@ -0,0 +1,69 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.eclipse.deeplearning4j.resources;
+
+/**
+ * Top level namespace for handling downloaded resources.
+ * Contains utility classes for working with different resources.
+ */
+public class DownloadResources {
+
+    /**
+     * Create a {@link CustomResource}
+     * @param fileName the name of the file to download
+     * @param localCacheDirectory the cache directory to store it
+     * @param rootUrl the root url of the download
+     * @return the created {@link CustomResource}
+     */
+    public static CustomResource createCustomResource(String fileName,String localCacheDirectory,String rootUrl) {
+        return new CustomResource(fileName,localCacheDirectory,rootUrl);
+    }
+
+    /**
+     * Create a legacy dl4j zoo model
+     * @param fileName the file name of the model to download
+     * @param modelName the name of the model to download
+     * @return
+     */
+    public static Dl4jZooResource createLegacyZooResource(String fileName,String modelName) {
+        return new Dl4jZooResource(fileName,modelName);
+    }
+
+    /**
+     * Create an {@link DataSetResource}
+     * @param fileName the name of the file to download
+     * @return
+     */
+    public static DataSetResource createDatasetResource(String fileName,String localCacheDirectory,String rootUrl) {
+        return new DataSetResource(fileName,localCacheDirectory,rootUrl);
+    }
+
+    /**
+     * Create a {@link OmniHubResource} from the given file
+     * @param fileName the name of the file to download
+     * @return the created resource
+     */
+    public static OmniHubResource createOmnihubResource(String fileName) {
+        return new OmniHubResource(fileName);
+    }
+
+
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/DownloadableResource.java

@@ -0,0 +1,134 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.resources;
+
+import lombok.SneakyThrows;
+import org.apache.commons.io.FileUtils;
+import org.nd4j.common.resources.Downloader;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+
+/**
+ * A downloadable resource represents a resource
+ * that is downloadable and cached in a specified
+ * directory. This will usually be in the user's home directory
+ * under a hidden dot directory.
+ */
+public interface DownloadableResource {
+
+
+    /**
+     * The name of the file to be downloaded
+     * or if an archive, the naem of the extracted file
+     * @return
+     */
+    String fileName();
+
+    /**
+     * The name of the archive file to be downloaded
+     * @return
+     */
+    String archiveFileName();
+
+
+    /**
+     * The root url to the resource.
+     * This is going to be the directory name of
+     * where the file lives.
+     * @return
+     */
+    String rootUrl();
+
+    /**
+     * The storage location representing the directory cache
+     * @return
+     */
+    File localCacheDirectory();
+
+    /**
+     * The local path to the file. This will usually be
+     * {@link #localCacheDirectory()} + {@link #fileName()}
+     * @return
+     */
+    default File localPath() {
+        return new File(localCacheDirectory(),fileName());
+    }
+    /**
+     * Md5sum of the file.
+     * Used for verification maybe empty
+     * to avoid running verification.
+     * @return
+     */
+    default String md5Sum() {
+        return "";
+    }
+
+
+
+
+    @SneakyThrows
+    default void download(boolean archive,int retries,int connectionTimeout,int readTimeout) {
+        if(archive) {
+            localCacheDirectory().mkdirs();
+            Downloader.downloadAndExtract(archiveFileName(),
+                    URI.create(rootUrl() + "/" + archiveFileName()).toURL(),
+                    new File(localCacheDirectory(),archiveFileName()),
+                    localPath(),
+                    md5Sum(),
+                    retries, connectionTimeout,
+                    readTimeout);
+        } else {
+            Downloader.download(fileName(),
+                    URI.create(rootUrl() + "/" + fileName()).toURL(),
+                    localPath(),
+                    md5Sum(),
+                    retries,
+                    connectionTimeout,
+                    readTimeout);
+        }
+    }
+
+
+    /**
+     * Returns the resource type for this resource
+     *
+     * @return
+     */
+    ResourceType resourceType();
+
+
+    /**
+     *
+     * @return
+     */
+    default boolean existsLocally() {
+        return localPath().exists();
+    }
+
+    default void delete() throws IOException {
+        if(localPath().isDirectory())
+            FileUtils.deleteDirectory(localPath());
+        else
+            FileUtils.forceDelete(localPath());
+    }
+
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/OmniHubResource.java

@@ -0,0 +1,54 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.resources;
+
+import org.deeplearning4j.omnihub.OmnihubConfig;
+
+import java.io.File;
+
+public class OmniHubResource extends BaseResource {
+
+
+    public OmniHubResource(String fileName) {
+        super(fileName,"");
+    }
+
+    @Override
+    public String fileName() {
+        return fileName;
+    }
+
+    @Override
+    public String rootUrl() {
+        return OmnihubConfig.getOmnihubUrl();
+    }
+
+    @Override
+    public File localCacheDirectory() {
+        return OmnihubConfig.getOmnihubHome();
+    }
+
+    @Override
+    public ResourceType resourceType() {
+        return ResourceType.OMNIHUB;
+    }
+
+
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/ResourceDataSets.java

@@ -0,0 +1,432 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.resources;
+
+import org.deeplearning4j.common.resources.DL4JResources;
+import org.deeplearning4j.common.resources.ResourceType;
+import org.eclipse.deeplearning4j.resources.utils.*;
+
+import java.io.File;
+
+import static org.eclipse.deeplearning4j.resources.utils.EMnistResourceConstants.*;
+import static org.eclipse.deeplearning4j.resources.utils.MnistResourceConstants.*;
+
+/**
+ * Top level class for leveraging pre curated datasets
+ * The pattern of usage for a default directory is:
+ *
+ * DataSetResource resource = ResourceDataSets.(..);
+ * For specifying a directory, for download it's:
+ * DataSetResource customDirResource = ResourceDataSets(yourCustomDirPath);
+ *
+ *  A user can then specify a resource download with:
+ *  resource.download(..);
+ *
+ *  A user won't normally use this class directly. It's common underlying
+ *  tooling for the pre existing dataset iterators such as
+ *  the MNISTDataSetIterator,EMnistDataSetIterator,LFWDataSetIterator
+ *  as well as the BaseImageLoader
+ *
+ * @author Adam Gibson
+ */
+public class ResourceDataSets {
+
+    private static File topLevelResourceDir = new File(System.getProperty("user.home"),".dl4jresources");
+
+
+    // --- Test files ---
+
+
+    /**
+     * The default top level directory for storing data
+     * @return
+     */
+    public static File topLevelResourceDir() {
+        return topLevelResourceDir;
+    }
+
+    /**
+     * Set the top level directory. If intended for use, this should be set
+     * before using any datasets or other resources.
+     * @param topLevelResourceDir the new directory to use
+     */
+    public static void setTopLevelResourceDir(File topLevelResourceDir) {
+        ResourceDataSets.topLevelResourceDir = topLevelResourceDir;
+    }
+
+
+    /**
+     * Download the training file for emnist mapping
+     * @param set the set to download
+     * @return the resource representing the subset
+     */
+    public static DataSetResource emnistMappingTrain(EMnistSet set) {
+        return emnistMapping(set,DL4JResources.getDirectory(ResourceType.DATASET, "EMNIST"),true);
+    }
+
+
+    /**
+     * The input mapping for the test set
+     * @param set
+     * @return
+     */
+    public static DataSetResource emnistMappingTest(EMnistSet set) {
+        return emnistMapping(set,DL4JResources.getDirectory(ResourceType.DATASET, "EMNIST"),false);
+    }
+
+    /**
+     * The input mapping for the training set
+     * @param set the emnist subset to use
+     * @param topLevelDir the top level directory to download to
+     * @return
+     */
+    public static DataSetResource emnistMappingTrain(EMnistSet set, File topLevelDir) {
+        return emnistMapping(set,topLevelDir,true);
+    }
+
+
+    /**
+     * The input label mapping for the test set
+     * @param set the subset of emnist to use
+     * @param topLevelDir the custom top level directory
+     * @return
+     */
+    public static DataSetResource emnistMappingTest(EMnistSet set, File topLevelDir) {
+        return emnistMapping(set,topLevelDir,false);
+    }
+
+
+    private static DataSetResource emnistMapping(EMnistSet set,File topLevelDir,boolean train) {
+        return new DataSetResource(
+                EMnistResourceConstants.getMappingFileName(set,train),
+                "",
+                null,
+                topLevelDir.getAbsolutePath(),
+                DL4JResources.getURLString("datasets/emnist")
+        );
+    }
+
+
+    /**
+     *
+     * @param set
+     * @param topLevelDir
+     * @return
+     */
+    public static DataSetResource emnistLabelsTrain(EMnistSet set, File topLevelDir) {
+        return emnistLabels(set,topLevelDir,true);
+    }
+
+
+    /**
+     * The emnist test labels
+     * @param set the emnist subset to download
+     * @param topLevelDir the top level directory to use
+     * @return
+     */
+    public static DataSetResource emnistLabelsTest(EMnistSet set, File topLevelDir) {
+        return emnistLabels(set,topLevelDir,false);
+    }
+
+    /**
+     * The emnist train labels
+     * @param set the emnist subset to download
+     * @return
+     */
+    public static DataSetResource emnistLabelsTrain(EMnistSet set) {
+        return emnistLabels(set,DL4JResources.getDirectory(ResourceType.DATASET, "EMNIST"),true);
+    }
+
+    /**
+     * The emnist test labels
+     * @param set the emnist subset to download
+     * @return
+     */
+    public static DataSetResource emnistLabelsTest(EMnistSet set) {
+        return emnistLabels(set,DL4JResources.getDirectory(ResourceType.DATASET, "EMNIST"),false);
+    }
+
+
+    private static DataSetResource emnistLabels(EMnistSet set,File topLevelDir,boolean train) {
+        return new DataSetResource(
+                getLabelsFileNameUnzipped(set,train),
+                train ? EMnistResourceConstants.getTrainingFileLabelsMD5(set) : EMnistResourceConstants.getTestFileLabelsMD5(set),
+                EMnistResourceConstants.getLabelsFileName(set,train),
+                topLevelDir.getAbsolutePath(),
+                DL4JResources.getURLString("datasets/emnist")
+        );
+    }
+
+    /**
+     * The emnist train labels
+     * @param set the emnist subset to download
+     * @param topLevelDir the top level directory to download to
+     * @return
+     */
+    public static DataSetResource emnistTrain(EMnistSet set,File topLevelDir) {
+        return emnist(set,topLevelDir,true);
+    }
+
+    /**
+     * The emnist test labels
+     * @param set the emnist subset to download
+     * @param topLevelDir the top level directory to download to
+     * @return
+     */
+    public static DataSetResource emnistTest(EMnistSet set,File topLevelDir) {
+        return emnist(set,topLevelDir,false);
+    }
+
+    /**
+     * The emnist test labels
+     * @param set the emnist subset to download
+     * @return
+     */
+    public static DataSetResource emnistTrain(EMnistSet set) {
+        return emnist(set, DL4JResources.getDirectory(ResourceType.DATASET, "EMNIST"),true);
+    }
+
+    /**
+     * The emnist test labels
+     * @param set the emnist subset to download
+     * @return
+     */
+    public static DataSetResource emnistTest(EMnistSet set) {
+        return emnist(set,DL4JResources.getDirectory(ResourceType.DATASET, "EMNIST"),false);
+    }
+
+
+    private static DataSetResource emnist(EMnistSet set,File topLevelDir,boolean train) {
+        return new DataSetResource(
+                EMnistResourceConstants.getImagesFileNameUnzipped(set,train),
+                train ? EMnistResourceConstants.getTrainingFilesMD5(set) : EMnistResourceConstants.getTestFilesMD5(set),
+                EMnistResourceConstants.getImagesFileName(set,train),
+                topLevelDir.getAbsolutePath(),
+                DL4JResources.getURLString("datasets/emnist")
+        );
+    }
+
+
+    /**
+     * The mnist training set
+     * @param topLevelDir the top level directory ot use
+     * @return
+     */
+    public static DataSetResource mnistTrain(File topLevelDir) {
+        return new DataSetResource(
+                MnistResourceConstants.getMNISTTrainingFilesFilename_unzipped(),
+                MnistResourceConstants.getMNISTTrainingFilesMD5(),
+                MnistResourceConstants.getMNISTTrainingFilesFilename(),
+                topLevelDir.getAbsolutePath(),
+                DL4JResources.getURLString(MnistResourceConstants.MNIST_ROOT)
+        );
+    }
+
+    /**
+     * The mnist training input data with a default directory of:
+     * {@link DL4JResources#getBaseDirectory()}
+     * @return
+     */
+    public static DataSetResource mnistTrain() {
+        return mnistTrain(DL4JResources.getBaseDirectory());
+    }
+
+
+    /**
+     * The mnist test images
+     * @param topLevelDir the top level directory to download to
+     * @return
+     */
+    public static DataSetResource mnistTest(File topLevelDir) {
+        return new DataSetResource(
+                MnistResourceConstants.getTestFilesFilename_unzipped(),
+                MnistResourceConstants.getTestFilesMD5(),
+                MnistResourceConstants.getTestFilesFilename(),
+                topLevelDir.getAbsolutePath(),
+                DL4JResources.getURLString(MnistResourceConstants.MNIST_ROOT)
+        );
+    }
+
+    /**
+     * The mnist test set input with a top level directory of:
+     * {@link DL4JResources#getBaseDirectory()}
+     * @return
+     */
+    public static DataSetResource mnistTest() {
+        return mnistTest(DL4JResources.getBaseDirectory());
+    }
+
+
+    /**
+     * The mnist training labels
+     * @param topLevelDir the top level directory to download to
+     * @return
+     */
+    public static DataSetResource mnistTrainLabels(File topLevelDir) {
+        return new DataSetResource(
+                MnistResourceConstants.getMNISTTrainingFileLabelsFilename_unzipped(),
+                MnistResourceConstants.getMNISTTrainingFileLabelsMD5(),
+                MnistResourceConstants.getMNISTTrainingFileLabelsFilename(),
+                topLevelDir.getAbsolutePath(),
+                DL4JResources.getURLString(MnistResourceConstants.MNIST_ROOT));
+    }
+
+    /**
+     * The mnist train labels with a top level directory of:
+     * {@link DL4JResources#getBaseDirectory()}
+     * @return
+     */
+    public static DataSetResource mnistTrainLabels() {
+        return mnistTrainLabels(DL4JResources.getBaseDirectory());
+    }
+
+
+    /**
+     * The mnist test labels
+     * @param topLevelDir the top level directory to download to
+     * @return
+     */
+    public static DataSetResource mnistTestLabels(File topLevelDir) {
+        return new DataSetResource(
+                MnistResourceConstants.getTestFileLabelsFilename_unzipped(),
+                MnistResourceConstants.getTestFileLabelsMD5(),
+                MnistResourceConstants.getTestFileLabelsFilename(),
+                topLevelDir.getAbsolutePath(),
+                DL4JResources.getURLString(MnistResourceConstants.MNIST_ROOT)
+        );
+    }
+
+
+    /**
+     * The mnist test set labels with a top level directory of:
+     * {@link DL4JResources#getBaseDirectory}
+     * @return
+     */
+    public static DataSetResource mnistTestLabels() {
+        return mnistTestLabels(DL4JResources.getBaseDirectory());
+    }
+
+
+
+
+    /**
+     *
+     * Labels for the LFW full dataset
+     * @param topLevelDir the top level directory to download to
+     * @return
+     */
+    public static DataSetResource lfwFullLabels(File topLevelDir) {
+        return new DataSetResource(
+                LFWResourceConstants.LFW_LABEL_FILE,
+                topLevelDir.getAbsolutePath(),
+                LFWResourceConstants.LFW_ROOT_URL
+        );
+    }
+
+    /**
+     *
+     * Input images for the LFW full dataset
+     * @param topLevelDir the top level directory to download to
+     * @return
+     */
+    public static DataSetResource lfwFullData(File topLevelDir) {
+        return new DataSetResource(
+                LFWResourceConstants.LFW_FULL_DIR,
+                LFWResourceConstants.LFW_DATA_URL,
+                topLevelDir.getAbsolutePath(),
+                LFWResourceConstants.LFW_ROOT_URL
+        );
+    }
+
+
+    /**
+     *
+     * Input images for the LFW smaller dataset
+     * @param topLevelDir the top level directory to download to
+     * @return
+     */
+    public static DataSetResource lfwSubData(File topLevelDir) {
+        return new DataSetResource(
+                LFWResourceConstants.LFW_SUB_DIR,
+                LFWResourceConstants.LFW_SUBSET_URL,
+                topLevelDir.getAbsolutePath(),
+                LFWResourceConstants.LFW_ROOT_URL
+        );
+    }
+
+
+    /**
+     * The cifar10 dataset
+     * @param topLevelDir the top level directory to download to
+     * @return
+     */
+    public static DataSetResource cifar10(File topLevelDir) {
+        return new DataSetResource(
+                CifarResourceConstants.CIFAR_DEFAULT_DIR.getAbsolutePath(),
+                CifarResourceConstants.CIFAR_ARCHIVE_FILE,
+                topLevelDir.getAbsolutePath(),
+                CifarResourceConstants.CIFAR_ROOT_URL);
+    }
+
+
+    /**
+     *
+     * Input labels for the LFW full dataset
+     * with a top level directory of:
+     * {@link LFWResourceConstants#LFW_FULL_DIR}
+     * @return
+     */
+    public static DataSetResource lfwFullLabels() {
+        return lfwFullLabels(new File(topLevelResourceDir, LFWResourceConstants.LFW_FULL_DIR));
+    }
+
+    /**
+     *
+     * Input images for the LFW full dataset
+     * with a top level directory of:
+     * {@link LFWResourceConstants#LFW_FULL_DIR}
+     * @return
+     */
+    public static DataSetResource lfwFullData() {
+        return lfwFullData(new File(topLevelResourceDir, LFWResourceConstants.LFW_FULL_DIR));
+    }
+
+    /**
+     *
+     * Input images for the LFW smaller dataset
+     * with a top level directory of:
+     * {@link LFWResourceConstants#LFW_SUB_DIR}
+     * @return
+     */
+    public static DataSetResource lfwSubData() {
+        return lfwSubData(new File(topLevelResourceDir, LFWResourceConstants.LFW_SUB_DIR));
+    }
+
+    /**
+     * The cifar 10 dataset with a default directory of:
+     * {@link CifarResourceConstants#CIFAR_DEFAULT_DIR}
+     * @return
+     */
+    public static DataSetResource cifar10() {
+        return cifar10(CifarResourceConstants.CIFAR_DEFAULT_DIR);
+    }
+
+
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/ResourceDirectory.java

@@ -0,0 +1,29 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.resources;
+
+import java.io.File;
+
+public class ResourceDirectory {
+
+    public final static String DEFAULT_PATH = new File(System.getProperty("user.home"),".dl4jresources").getAbsolutePath();
+
+
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/ResourceType.java

@@ -0,0 +1,28 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.resources;
+
+public enum ResourceType {
+    LEGACY_ZOO,
+    OMNIHUB,
+    STRUMPF,
+    DATASET,
+    CUSTOM
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/StrumpfResource.java

@@ -0,0 +1,54 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.resources;
+
+import org.nd4j.common.resources.strumpf.StrumpfResolver;
+
+import java.io.File;
+
+public class StrumpfResource extends BaseResource {
+    private StrumpfResolver resolver = new StrumpfResolver();
+
+    public StrumpfResource(String fileName) {
+        super(fileName,"");
+    }
+
+    @Override
+    public String fileName() {
+        return fileName;
+    }
+
+    @Override
+    public String rootUrl() {
+        return "resolver";
+    }
+
+    @Override
+    public File localCacheDirectory() {
+        return resolver.localCacheRoot();
+    }
+
+    @Override
+    public ResourceType resourceType() {
+        return ResourceType.STRUMPF;
+    }
+
+
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/utils/CifarResourceConstants.java

@@ -0,0 +1,32 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.resources.utils;
+
+import org.apache.commons.io.FilenameUtils;
+import org.eclipse.deeplearning4j.resources.ResourceDataSets;
+
+import java.io.File;
+
+public class CifarResourceConstants {
+    public static final String CIFAR_ROOT_URL = "https://www.cs.toronto.edu/~kriz";
+    public final static String CIFAR_ARCHIVE_FILE = "cifar-10-binary.tar.gz";
+    public final static File CIFAR_DEFAULT_DIR =  new File(ResourceDataSets.topLevelResourceDir(),
+            FilenameUtils.concat("cifar", "cifar-10-batches-bin"));
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/utils/EMnistResourceConstants.java

@@ -0,0 +1,218 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.resources.utils;
+
+import org.deeplearning4j.common.resources.DL4JResources;
+
+public class EMnistResourceConstants {
+
+    public static String getTrainingFilesURL(EMnistSet ds) {
+        return DL4JResources.getURLString("datasets/emnist/" + getImagesFileName(ds, true));
+    }
+
+    public static String getTrainingFilesMD5(EMnistSet ds) {
+        switch (ds) {
+            case COMPLETE:
+                //byclass-train-images
+                return "712dda0bd6f00690f32236ae4325c377";
+            case MERGE:
+                //bymerge-train-images
+                return "4a792d4df261d7e1ba27979573bf53f3";
+            case BALANCED:
+                //balanced-train-images
+                return "4041b0d6f15785d3fa35263901b5496b";
+            case LETTERS:
+                //letters-train-images
+                return "8795078f199c478165fe18db82625747";
+            case DIGITS:
+                //digits-train-images
+                return "d2662ecdc47895a6bbfce25de9e9a677";
+            case MNIST:
+                //mnist-train-images
+                return "3663598a39195d030895b6304abb5065";
+            default:
+                throw new UnsupportedOperationException("Unknown DataSet: " + ds);
+        }
+    }
+
+    public static String getTrainingFilesFilename(EMnistSet ds) {
+        return getImagesFileName(ds, true);
+    }
+
+    public static String getTrainingFilesFilename_unzipped(EMnistSet ds) {
+        return getImagesFileNameUnzipped(ds, true);
+    }
+
+    public static String getTrainingFileLabelsURL(EMnistSet ds) {
+        return DL4JResources.getURLString("datasets/emnist/" + getLabelsFileName(ds, true));
+    }
+
+    public static String getTrainingFileLabelsMD5(EMnistSet ds) {
+        switch (ds) {
+            case COMPLETE:
+                //byclass-train-labels
+                return "ee299a3ee5faf5c31e9406763eae7e43";
+            case MERGE:
+                //bymerge-train-labels
+                return "491be69ef99e1ab1f5b7f9ccc908bb26";
+            case BALANCED:
+                //balanced-train-labels
+                return "7a35cc7b2b7ee7671eddf028570fbd20";
+            case LETTERS:
+                //letters-train-labels
+                return "c16de4f1848ddcdddd39ab65d2a7be52";
+            case DIGITS:
+                //digits-train-labels
+                return "2223fcfee618ac9c89ef20b6e48bcf9e";
+            case MNIST:
+                //mnist-train-labels
+                return "6c092f03c9bb63e678f80f8bc605fe37";
+            default:
+                throw new UnsupportedOperationException("Unknown DataSet: " + ds);
+        }
+    }
+
+    public static String getTrainingFileLabelsFilename(EMnistSet ds) {
+        return getLabelsFileName(ds, true);
+    }
+
+    public static String getTrainingFileLabelsFilename_unzipped(EMnistSet ds) {
+        return getLabelsFileNameUnzipped(ds, true);
+    }
+
+
+    // --- Test files ---
+
+    public static String getTestFilesURL(EMnistSet ds) {
+        return DL4JResources.getURLString("datasets/emnist/" + getImagesFileName(ds, false));
+    }
+
+    public static String getTestFilesMD5(EMnistSet ds) {
+        switch (ds) {
+            case COMPLETE:
+                //byclass-test-images
+                return "1435209e34070a9002867a9ab50160d7";
+            case MERGE:
+                //bymerge-test-images
+                return "8eb5d34c91f1759a55831c37ec2a283f";
+            case BALANCED:
+                //balanced-test-images
+                return "6818d20fe2ce1880476f747bbc80b22b";
+            case LETTERS:
+                //letters-test-images
+                return "382093a19703f68edac6d01b8dfdfcad";
+            case DIGITS:
+                //digits-test-images
+                return "a159b8b3bd6ab4ed4793c1cb71a2f5cc";
+            case MNIST:
+                //mnist-test-images
+                return "fb51b6430fc4dd67deaada1bf25d4524";
+            default:
+                throw new UnsupportedOperationException("Unknown DataSet: " + ds);
+        }
+    }
+
+    public static String getTestFilesFilename(EMnistSet ds) {
+        return getImagesFileName(ds, false);
+    }
+
+    public static String getTestFilesFilename_unzipped(EMnistSet ds) {
+        return getImagesFileNameUnzipped(ds, false);
+    }
+
+    public static String getTestFileLabelsURL(EMnistSet ds) {
+        return DL4JResources.getURLString("datasets/emnist/" + getLabelsFileName(ds, false));
+    }
+
+    public static String getTestFileLabelsMD5(EMnistSet ds) {
+        switch (ds) {
+            case COMPLETE:
+                //byclass-test-labels
+                return "7a0f934bd176c798ecba96b36fda6657";
+            case MERGE:
+                //bymerge-test-labels
+                return "c13f4cd5211cdba1b8fa992dae2be992";
+            case BALANCED:
+                //balanced-test-labels
+                return "acd3694070dcbf620e36670519d4b32f";
+            case LETTERS:
+                //letters-test-labels
+                return "d4108920cd86601ec7689a97f2de7f59";
+            case DIGITS:
+                //digits-test-labels
+                return "8afde66ea51d865689083ba6bb779fac";
+            case MNIST:
+                //mnist-test-labels
+                return "ae7f6be798a9a5d5f2bd32e078a402dd";
+            default:
+                throw new UnsupportedOperationException("Unknown DataSet: " + ds);
+        }
+    }
+
+    public static String getTestFileLabelsFilename(EMnistSet ds) {
+        return getLabelsFileName(ds, false);
+    }
+
+    public static String getTestFileLabelsFilename_unzipped(EMnistSet ds) {
+        return getLabelsFileNameUnzipped(ds, false);
+    }
+
+
+    public static String getImagesFileName(EMnistSet ds, boolean train) {
+        return "emnist-" + name(ds) + "-" + (train ? "train" : "test") + "-images-idx3-ubyte.gz";
+    }
+
+    public static String getImagesFileNameUnzipped(EMnistSet ds, boolean train) {
+        return "emnist-" + name(ds) + "-" + (train ? "train" : "test") + "-images-idx3-ubyte";
+    }
+
+    public static String getLabelsFileName(EMnistSet ds, boolean train) {
+        return "emnist-" + name(ds) + "-" + (train ? "train" : "test") + "-labels-idx1-ubyte.gz";
+    }
+
+    public static String getLabelsFileNameUnzipped(EMnistSet ds, boolean train) {
+        return "emnist-" + name(ds) + "-" + (train ? "train" : "test") + "-labels-idx1-ubyte";
+    }
+
+    public static String getMappingFileName(EMnistSet ds, boolean train) {
+        return "emnist-" + name(ds) + "-mapping.txt";
+    }
+
+    public static String name(EMnistSet ds) {
+        switch (ds) {
+            case COMPLETE:
+                return "byclass";
+            case MERGE:
+                return "bymerge";
+            case BALANCED:
+                return "balanced";
+            case LETTERS:
+                return "letters";
+            case DIGITS:
+                return "digits";
+            case MNIST:
+                return "mnist";
+            default:
+                throw new UnsupportedOperationException("Unknown DataSet: " + ds);
+        }
+    }
+
+
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/utils/EMnistSet.java

@@ -0,0 +1,27 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.resources.utils;
+
+/**
+ * EMNIST dataset has multiple different subsets
+ */
+public enum EMnistSet {
+    COMPLETE, MERGE, BALANCED, LETTERS, DIGITS, MNIST
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/utils/LFWResourceConstants.java

@@ -0,0 +1,32 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.resources.utils;
+
+public class LFWResourceConstants {
+
+
+    public final static String LFW_ROOT_URL = "http://vis-www.cs.umass.edu/lfw";
+    public final static String LFW_DATA_URL = "lfw.tgz";
+    public final static String LFW_LABEL_URL = "lfw-names.txt";
+    public final static String LFW_SUBSET_URL = "lfw-a.tgz";
+    public final static String LFW_FULL_DIR = "lfw";
+    public final static String LFW_SUB_DIR = "lfw-a";
+    public final static String LFW_LABEL_FILE = "lfw-names.txt";
+}

resources/src/main/java/org/eclipse/deeplearning4j/resources/utils/MnistResourceConstants.java

@@ -0,0 +1,110 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.resources.utils;
+
+import org.deeplearning4j.common.resources.DL4JResources;
+
+public class MnistResourceConstants {
+
+    public static final String TRAINING_FILES_URL_RELATIVE = "datasets/mnist/train-images-idx3-ubyte.gz";
+    public static final String TRAINING_FILES_MD_5 = "f68b3c2dcbeaaa9fbdd348bbdeb94873";
+    public static final String TRAINING_FILES_FILENAME = "train-images-idx3-ubyte.gz";
+    public static final String TRAINING_FILES_FILENAME_UNZIPPED = "train-images-idx3-ubyte";
+    public static final String TRAINING_FILE_LABELS_URL_RELATIVE = "datasets/mnist/train-labels-idx1-ubyte.gz";
+    public static final String TRAINING_FILE_LABELS_MD_5 = "d53e105ee54ea40749a09fcbcd1e9432";
+    public static final String TRAINING_FILE_LABELS_FILENAME = "train-labels-idx1-ubyte.gz";
+    public static final String TRAINING_FILE_LABELS_FILENAME_UNZIPPED = "train-labels-idx1-ubyte";
+    //Test data:
+    public static final String TEST_FILES_URL_RELATIVE = "datasets/mnist/t10k-images-idx3-ubyte.gz";
+    public static final String TEST_FILES_MD_5 = "9fb629c4189551a2d022fa330f9573f3";
+    public static final String TEST_FILES_FILENAME = "t10k-images-idx3-ubyte.gz";
+    public static final String TEST_FILES_FILENAME_UNZIPPED = "t10k-images-idx3-ubyte";
+    public static final String TEST_FILE_LABELS_URL_RELATIVE = "datasets/mnist/t10k-labels-idx1-ubyte.gz";
+    public static final String TEST_FILE_LABELS_MD_5 = "ec29112dd5afa0611ce80d1b7f02629c";
+    public static final String TEST_FILE_LABELS_FILENAME = "t10k-labels-idx1-ubyte.gz";
+    public static final String TEST_FILE_LABELS_FILENAME_UNZIPPED = "t10k-labels-idx1-ubyte";
+
+    public final static String MNIST_ROOT = "datasets/mnist";
+
+    // --- Train files ---
+    public static String getMNISTTrainingFilesURL() {
+        return DL4JResources.getURLString(TRAINING_FILES_URL_RELATIVE);
+    }
+
+    public static String getMNISTTrainingFilesMD5() {
+        return TRAINING_FILES_MD_5;
+    }
+
+    public static String getMNISTTrainingFilesFilename() {
+        return TRAINING_FILES_FILENAME;
+    }
+
+    public static String getMNISTTrainingFilesFilename_unzipped() {
+        return TRAINING_FILES_FILENAME_UNZIPPED;
+    }
+
+    public static String getMNISTTrainingFileLabelsURL() {
+        return DL4JResources.getURLString(TRAINING_FILE_LABELS_URL_RELATIVE);
+    }
+
+    public static String getMNISTTrainingFileLabelsMD5() {
+        return TRAINING_FILE_LABELS_MD_5;
+    }
+
+    public static String getMNISTTrainingFileLabelsFilename() {
+        return TRAINING_FILE_LABELS_FILENAME;
+    }
+
+    public static String getMNISTTrainingFileLabelsFilename_unzipped() {
+        return TRAINING_FILE_LABELS_FILENAME_UNZIPPED;
+    }
+
+    public static String getTestFilesURL() {
+        return DL4JResources.getURLString(TEST_FILES_URL_RELATIVE);
+    }
+
+    public static String getTestFilesMD5() {
+        return TEST_FILES_MD_5;
+    }
+
+    public static String getTestFilesFilename() {
+        return TEST_FILES_FILENAME;
+    }
+
+    public static String getTestFilesFilename_unzipped() {
+        return TEST_FILES_FILENAME_UNZIPPED;
+    }
+
+    public  static String getTestFileLabelsURL() {
+        return DL4JResources.getURLString(TEST_FILE_LABELS_URL_RELATIVE);
+    }
+
+    public static String getTestFileLabelsMD5() {
+        return TEST_FILE_LABELS_MD_5;
+    }
+
+    public static String getTestFileLabelsFilename() {
+        return TEST_FILE_LABELS_FILENAME;
+    }
+
+    public static String getTestFileLabelsFilename_unzipped() {
+        return TEST_FILE_LABELS_FILENAME_UNZIPPED;
+    }
+}

resources/src/main/java/org/nd4j/common/resources/Downloader.java

@@ -0,0 +1,224 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.common.resources;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.nd4j.common.util.ArchiveUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.RandomAccessFile;
+import java.net.URL;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.FileLockInterruptionException;
+import java.nio.channels.OverlappingFileLockException;
+
+@Slf4j
+public class Downloader {
+    /**
+     * Default connection timeout in milliseconds when using {@link FileUtils#copyURLToFile(URL, File, int, int)}
+     */
+    public static final int DEFAULT_CONNECTION_TIMEOUT = 60000;
+    /**
+     * Default read timeout in milliseconds when using {@link FileUtils#copyURLToFile(URL, File, int, int)}
+     */
+    public static final int DEFAULT_READ_TIMEOUT = 60000;
+
+    private Downloader(){ }
+
+    /**
+     * As per {@link #download(String, URL, File, String, int, int, int)} with the connection and read timeouts
+     * set to their default values - {@link #DEFAULT_CONNECTION_TIMEOUT} and {@link #DEFAULT_READ_TIMEOUT} respectively
+     */
+    public static void download(String name, URL url, File f, String targetMD5, int maxTries) throws IOException {
+        download(name, url, f, targetMD5, maxTries, DEFAULT_CONNECTION_TIMEOUT, DEFAULT_READ_TIMEOUT);
+    }
+
+    /**
+     * Download the specified URL to the specified file, and verify that the target MD5 matches
+     *
+     * @param name              Name (mainly for providing useful exceptions)
+     * @param url               URL to download
+     * @param f                 Destination file
+     * @param targetMD5         Expected MD5 for file
+     * @param maxTries          Maximum number of download attempts before failing and throwing an exception
+     * @param connectionTimeout connection timeout in milliseconds, as used by {@link FileUtils#copyURLToFile(URL, File, int, int)}
+     * @param readTimeout       read timeout in milliseconds, as used by {@link FileUtils#copyURLToFile(URL, File, int, int)}
+     * @throws IOException If an error occurs during downloading
+     */
+    public static void download(String name, URL url, File f, String targetMD5, int maxTries, int connectionTimeout, int readTimeout) throws IOException {
+        download(name, url, f, targetMD5, maxTries, 0, connectionTimeout, readTimeout);
+    }
+
+    private static void download(String name, URL url, File f, String targetMD5, int maxTries, int attempt, int connectionTimeout, int readTimeout) throws IOException {
+        doOrWait(f.getParentFile(), () -> {
+            boolean isCorrectFile = f.exists() && f.isFile() && checkMD5OfFile(targetMD5, f);
+            if (attempt < maxTries) {
+                if(!isCorrectFile) {
+                    FileUtils.copyURLToFile(url, f, connectionTimeout, readTimeout);
+                    if (!checkMD5OfFile(targetMD5, f)) {
+                        f.delete();
+                        download(name, url, f, targetMD5, maxTries, attempt + 1, connectionTimeout, readTimeout);
+                    }
+                }
+            } else if (!isCorrectFile) {
+                //Too many attempts
+                throw new IOException("Could not download " + name + " from " + url + "\n properly despite trying " + maxTries
+                        + " times, check your connection.");
+            }
+        });
+
+    }
+
+    /**
+     * As per {@link #downloadAndExtract(String, URL, File, File, String, int, int, int)} with the connection and read timeouts
+     *      * set to their default values - {@link #DEFAULT_CONNECTION_TIMEOUT} and {@link #DEFAULT_READ_TIMEOUT} respectively
+     */
+    public static void downloadAndExtract(String name, URL url, File f, File extractToDir, String targetMD5, int maxTries) throws IOException {
+        downloadAndExtract(name, url, f, extractToDir, targetMD5, maxTries, DEFAULT_CONNECTION_TIMEOUT, DEFAULT_READ_TIMEOUT);
+    }
+
+    /**
+     * Download the specified URL to the specified file, verify that the MD5 matches, and then extract it to the specified directory.<br>
+     * Note that the file must be an archive, with the correct file extension: .zip, .jar, .tar.gz, .tgz or .gz
+     *
+     * @param name         Name (mainly for providing useful exceptions)
+     * @param url          URL to download
+     * @param f            Destination file
+     * @param extractToDir Destination directory to extract all files
+     * @param targetMD5    Expected MD5 for file
+     * @param maxTries     Maximum number of download attempts before failing and throwing an exception
+     * @param connectionTimeout connection timeout in milliseconds, as used by {@link FileUtils#copyURLToFile(URL, File, int, int)}
+     * @param readTimeout       read timeout in milliseconds, as used by {@link FileUtils#copyURLToFile(URL, File, int, int)}
+     * @throws IOException If an error occurs during downloading
+     */
+    public static void downloadAndExtract(String name, URL url, File f, File extractToDir, String targetMD5, int maxTries,
+                                          int connectionTimeout, int readTimeout) throws IOException {
+        downloadAndExtract(0, maxTries, name, url, f, extractToDir, targetMD5, connectionTimeout, readTimeout);
+    }
+
+    private static void downloadAndExtract(int attempt, int maxTries, String name, URL url, File f, File extractToDir,
+                                           String targetMD5, int connectionTimeout, int readTimeout) throws IOException {
+        doOrWait(f.getParentFile(), () -> {
+            boolean isCorrectFile = f.exists() && f.isFile() && checkMD5OfFile(targetMD5, f);
+            if (attempt < maxTries) {
+                if(!isCorrectFile) {
+                    FileUtils.copyURLToFile(url, f, connectionTimeout, readTimeout);
+                    if (!checkMD5OfFile(targetMD5, f)) {
+                        f.delete();
+                        downloadAndExtract(attempt + 1, maxTries, name, url, f, extractToDir, targetMD5, connectionTimeout, readTimeout);
+                    }
+                }
+                // try extracting
+                try{
+                    ArchiveUtils.unzipFileTo(f.getAbsolutePath(), extractToDir.getAbsolutePath(), false);
+                } catch (Throwable t){
+                    log.warn("Error extracting {} files from file {} - retrying...", name, f.getAbsolutePath(), t);
+                    f.delete();
+                    downloadAndExtract(attempt + 1, maxTries, name, url, f, extractToDir, targetMD5, connectionTimeout, readTimeout);
+                }
+            } else if (!isCorrectFile) {
+                //Too many attempts
+                throw new IOException("Could not download and extract " + name + " from " + url.getPath() + "\n properly despite trying " + maxTries
+                        + " times, check your connection. File info:" + "\nTarget MD5: " + targetMD5
+                        + "\nHash matches: " + checkMD5OfFile(targetMD5, f) + "\nIs valid file: " + f.isFile());
+            }
+        });
+    }
+
+
+    /**
+     * Check if the  input file is corrupted by
+     * verifying the target md5 hash.
+     * If corrupted, file is deleted
+     * @param inputFile the input file to check
+     * @param targetmd5 the target md5 to test
+     * @throws IOException
+     */
+    public static boolean deleteIfCorrupted(File inputFile,String targetmd5) throws IOException {
+        if(!checkMD5OfFile(targetmd5, inputFile)) {
+            inputFile.delete();
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * Check the MD5 of the specified file
+     * @param targetMD5 Expected MD5
+     * @param file      File to check
+     * @return          True if MD5 matches, false otherwise
+     */
+    public static boolean checkMD5OfFile(String targetMD5, File file) throws IOException {
+       if(!file.exists())
+           return false;
+
+        if(targetMD5.isEmpty())
+           return true;
+
+        InputStream in = FileUtils.openInputStream(file);
+        String trueMd5 = DigestUtils.md5Hex(in);
+        IOUtils.closeQuietly(in);
+        return (targetMD5.equals(trueMd5));
+    }
+
+    private static void doOrWait(File flagDir, IOCallable block) throws IOException {
+        boolean waitForFinish = false;
+        if(flagDir.exists()){
+            final File lockFile = flagDir.toPath().resolve("inProgress.lock").toFile();
+            RandomAccessFile flag = new RandomAccessFile(lockFile, "rw");
+            while(true) try {
+                final FileChannel channel = flag.getChannel();
+                try (FileLock lock = channel.lock()) {
+                    if(!waitForFinish) block.call();
+                } finally {
+                    lockFile.delete();
+                }
+                return;
+            }catch(OverlappingFileLockException | FileLockInterruptionException e){
+                // file is locked, someone else is already doing the work we want to do.
+                // just wait until it is finished, there should be no need to actually do anything
+                // once we can acquire that lock
+                try {
+                    log.debug("Waiting to acquire download lock in dir {}", flagDir.getPath());
+                    waitForFinish = true;
+                    Thread.sleep(100);
+                } catch (InterruptedException ignored) {
+                    // noop, we retry to acquire that lock
+                }
+            }
+        }else{
+            throw new IOException("Target directory "+flagDir.getPath()+" must exist!");
+        }
+    }
+
+    @FunctionalInterface
+    private interface IOCallable {
+        void call() throws IOException;
+    }
+
+}

resources/src/main/java/org/nd4j/common/resources/Resolver.java

@@ -0,0 +1,92 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.common.resources;
+
+import java.io.File;
+import java.io.InputStream;
+
+public interface Resolver {
+
+    /**
+     * Priority of this resolver. 0 is highest priority (check first), larger values are lower priority (check last)
+     */
+    int priority();
+
+    /**
+     * Determine if the specified file resource can be resolved by {@link #asFile(String)} and {@link #asStream(String)}
+     *
+     * @param resourcePath Path of the resource to be resolved
+     * @return True if this resolver is able to resolve the resource file - i.e., whether it is a valid path and exists
+     */
+    boolean exists(String resourcePath);
+
+    /**
+     * Determine if the specified directory resource can be resolved by {@link #copyDirectory(String, File)}
+     *
+     * @param dirPath Path of the directory resource to be resolved
+     * @return True if this resolver is able to resolve the directory - i.e., whether it is a valid path and exists
+     */
+    boolean directoryExists(String dirPath);
+
+    /**
+     * Get the specified resources as a standard local file.
+     * Note that the resource must exist as determined by {@link #exists(String)}
+     *
+     * @param resourcePath Path of the resource.
+     * @return The local file version of the resource
+     */
+    File asFile(String resourcePath);
+
+    /**
+     * Get the specified resources as an input stream.
+     * Note that the resource must exist as determined by {@link #exists(String)}
+     *
+     * @param resourcePath Path of the resource.
+     * @return The resource as an input stream
+     */
+    InputStream asStream(String resourcePath);
+
+    /**
+     * Copy the directory resource (recursively) to the specified destination directory
+     *
+     * @param dirPath        Path of the resource directory to resolve
+     * @param destinationDir Where the files should be copied to
+     */
+    void copyDirectory(String dirPath, File destinationDir);
+
+    /**
+     * @return True if the resolver has a local cache directory, as returned by {@link #localCacheRoot()}
+     */
+    boolean hasLocalCache();
+
+    /**
+     * @return Root directory of the local cache, or null if {@link #hasLocalCache()} returns false
+     */
+    File localCacheRoot();
+
+    /**
+     * Normalize the path that may be a resource reference.
+     * For example: "someDir/myFile.zip.resource_reference" --> "someDir/myFile.zip"
+     * Returns null if the file cannot be resolved.
+     * If the file is not a reference, the original path is returned
+     */
+    String normalizePath(String path);
+}

resources/src/main/java/org/nd4j/common/resources/Resources.java

@@ -0,0 +1,152 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.common.resources;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.nd4j.common.config.ND4JClassLoading;
+import org.nd4j.common.resources.strumpf.StrumpfResolver;
+
+import java.io.File;
+import java.io.InputStream;
+import java.util.*;
+
+@Slf4j
+public class Resources {
+    private static Resources INSTANCE = new Resources();
+
+    protected final List<Resolver> resolvers;
+
+    protected Resources() {
+        ServiceLoader<Resolver> loader = ND4JClassLoading.loadService(Resolver.class);
+
+        resolvers = new ArrayList<>();
+        resolvers.add(new StrumpfResolver());
+        for (Resolver resolver : loader) {
+            resolvers.add(resolver);
+        }
+
+        //Sort resolvers by priority: check resolvers with lower numbers first
+        Collections.sort(resolvers, Comparator.comparingInt(Resolver::priority));
+    }
+
+    /**
+     * Check if the specified resource exists (can be resolved by any method) hence can be loaded by {@link #asFile(String)}
+     * or {@link #asStream(String)}
+     *
+     * @param resourcePath Path of the resource to be resolved
+     * @return Whether the resource can be resolved or not
+     */
+    public static boolean exists(@NonNull String resourcePath) {
+        return INSTANCE.resourceExists(resourcePath);
+    }
+
+    /**
+     * Get the specified resource as a local file.
+     * If it cannot be found (i.e., {@link #exists(String)} returns false) this method will throw an exception.
+     *
+     * @param resourcePath Path of the resource to get
+     * @return Resource file
+     */
+    public static File asFile(@NonNull String resourcePath) {
+        return INSTANCE.getAsFile(resourcePath);
+    }
+
+    /**
+     * Get the specified resource as an input stream.<br>
+     * If it cannot be found (i.e., {@link #exists(String)} returns false) this method will throw an exception.
+     *
+     * @param resourcePath Path of the resource to get
+     * @return Resource stream
+     */
+    public static InputStream asStream(@NonNull String resourcePath) {
+        return INSTANCE.getAsStream(resourcePath);
+    }
+
+    /**
+     * Copy the contents of the specified directory (path) to the specified destination directory, resolving any resources in the process
+     *
+     * @param directoryPath  Directory to copy contents of
+     * @param destinationDir Destination
+     */
+    public static void copyDirectory(@NonNull String directoryPath, @NonNull File destinationDir) {
+        INSTANCE.copyDir(directoryPath, destinationDir);
+    }
+
+    /**
+     * Normalize the path that may be a resource reference.
+     * For example: "someDir/myFile.zip.resource_reference" --> "someDir/myFile.zip"
+     * Returns null if the file cannot be resolved.
+     * If the file is not a reference, the original path is returned
+     */
+    public static String normalizePath(String path){
+        return INSTANCE.normalize(path);
+    }
+
+    protected boolean resourceExists(String resourcePath) {
+        for (Resolver r : resolvers) {
+            if (r.exists(resourcePath))
+                return true;
+        }
+
+        return false;
+    }
+
+    protected File getAsFile(String resourcePath) {
+        for (Resolver r : resolvers) {
+            if (r.exists(resourcePath)) {
+                return r.asFile(resourcePath);
+            }
+        }
+
+        throw new IllegalStateException("Cannot resolve resource (not found): none of " + resolvers.size() +
+                " resolvers can resolve resource \"" + resourcePath + "\" - available resolvers: " + resolvers.toString());
+    }
+
+    public InputStream getAsStream(String resourcePath) {
+        for (Resolver r : resolvers) {
+            if (r.exists(resourcePath)) {
+                log.debug("Resolved resource with resolver " + r.getClass().getName() + " for path " + resourcePath);
+                return r.asStream(resourcePath);
+            }
+        }
+
+        throw new IllegalStateException("Cannot resolve resource (not found): none of " + resolvers.size() +
+                " resolvers can resolve resource \"" + resourcePath + "\" - available resolvers: " + resolvers.toString());
+    }
+
+    public void copyDir(String directoryPath, File destinationDir) {
+        for (Resolver r : resolvers) {
+            if (r.directoryExists(directoryPath)) {
+                r.copyDirectory(directoryPath, destinationDir);
+                return;
+            }
+        }
+    }
+
+    public String normalize(String path){
+        for(Resolver r : resolvers){
+            path = r.normalizePath(path);
+        }
+        return path;
+    }
+
+}

resources/src/main/java/org/nd4j/common/resources/strumpf/ResourceFile.java

@@ -0,0 +1,259 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.common.resources.strumpf;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.io.IOUtils;
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.common.config.ND4JSystemProperties;
+import org.nd4j.shade.guava.io.Files;
+import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties;
+import org.nd4j.shade.jackson.databind.DeserializationFeature;
+import org.nd4j.shade.jackson.databind.MapperFeature;
+import org.nd4j.shade.jackson.databind.ObjectMapper;
+import org.nd4j.shade.jackson.databind.SerializationFeature;
+
+import java.io.*;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+@AllArgsConstructor
+@NoArgsConstructor
+@Data
+@JsonIgnoreProperties("filePath")
+@Slf4j
+public class ResourceFile {
+    /**
+     * Default value for resource downloading connection timeout - see {@link ND4JSystemProperties#RESOURCES_CONNECTION_TIMEOUT}
+     */
+    public static final int DEFAULT_CONNECTION_TIMEOUT = 60000;        //Timeout for connections to be established
+    /**
+     * Default value for resource downloading read timeout - see {@link ND4JSystemProperties#RESOURCES_READ_TIMEOUT}
+     */
+    public static final int DEFAULT_READ_TIMEOUT = 60000;              //Timeout for amount of time between connection established and data is available
+    protected static final String PATH_KEY = "full_remote_path";
+    protected static final String HASH = "_hash";
+    protected static final String COMPRESSED_HASH = "_compressed_hash";
+
+    protected static final int MAX_DOWNLOAD_ATTEMPTS = 3;
+
+    public static final ObjectMapper MAPPER = newMapper();
+
+    //Note: Field naming to match Strumpf JSON format
+    protected int current_version;
+    protected Map<String, String> v1;
+
+    //Not in JSON:
+    protected String filePath;
+
+    public static ResourceFile fromFile(String path) {
+        return fromFile(new File(path));
+    }
+
+    public static ResourceFile fromFile(File file) {
+        String s;
+        try {
+            s = FileUtils.readFileToString(file, StandardCharsets.UTF_8);
+            ResourceFile rf = MAPPER.readValue(s, ResourceFile.class);
+            rf.setFilePath(file.getPath());
+            return rf;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public String relativePath() {
+        String hashKey = null;
+        for (String key : v1.keySet()) {
+            if (key.endsWith(HASH) && !key.endsWith(COMPRESSED_HASH)) {
+                hashKey = key;
+                break;
+            }
+        }
+        if (hashKey == null) {
+            throw new IllegalStateException("Could not find <filename>_hash in resource reference file: " + filePath);
+        }
+
+        String relativePath = hashKey.substring(0, hashKey.length() - 5); //-5 to remove "_hash" suffix
+        return relativePath.replaceAll("\\\\", "/");
+    }
+
+    public boolean localFileExistsAndValid(File cacheRootDir) {
+
+        File file = getLocalFile(cacheRootDir);
+        if (!file.exists()) {
+            return false;
+        }
+
+        //File exists... but is it valid?
+        String sha256Property = relativePath() + HASH;
+        String expSha256 = v1.get(sha256Property);
+
+        Preconditions.checkState(expSha256 != null, "Expected JSON property %s was not found in resource reference file %s", sha256Property, filePath);
+
+        String actualSha256 = sha256(file);
+        if (!expSha256.equals(actualSha256)) {
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Get the local file - or where it *would* be if it has been downloaded. If it does not exist, it will not be downloaded here
+     *
+     * @return
+     */
+    protected File getLocalFile(File cacheRootDir) {
+        String relativePath = relativePath();
+
+        //For resolving local files with different versions, we want paths like:
+        // ".../dir/filename.txt__v1/filename.txt"
+        // ".../dir/filename.txt__v2/filename.txt"
+        //This is to support multiple versions of files simultaneously... for example, different projects needing different
+        // versions, or supporting old versions of resource files etc
+
+        int lastSlash = Math.max(relativePath.lastIndexOf('/'), relativePath.lastIndexOf('\\'));
+        String filename;
+        if (lastSlash < 0) {
+            filename = relativePath;
+        } else {
+            filename = relativePath.substring(lastSlash + 1);
+        }
+
+        File parentDir = new File(cacheRootDir, relativePath + "__v" + current_version);
+        File file = new File(parentDir, filename);
+        return file;
+    }
+
+    /**
+     * Get the local file - downloading and caching if required
+     *
+     * @return
+     */
+    public File localFile(File cacheRootDir) {
+        if (localFileExistsAndValid(cacheRootDir)) {
+            return getLocalFile(cacheRootDir);
+        }
+
+        //Need to download and extract...
+        String remotePath = v1.get(PATH_KEY);
+        Preconditions.checkState(remotePath != null, "No remote path was found in resource reference file %s", filePath);
+        File f = getLocalFile(cacheRootDir);
+
+        File tempDir = Files.createTempDir();
+        File tempFile = new File(tempDir, FilenameUtils.getName(remotePath));
+
+        String sha256PropertyCompressed = relativePath() + COMPRESSED_HASH;
+
+        String sha256Compressed = v1.get(sha256PropertyCompressed);
+        Preconditions.checkState(sha256Compressed != null, "Expected JSON property %s was not found in resource reference file %s", sha256PropertyCompressed, filePath);
+
+        String sha256Property = relativePath() + HASH;
+        String sha256Uncompressed = v1.get(sha256Property);
+
+        String connTimeoutStr = System.getProperty(ND4JSystemProperties.RESOURCES_CONNECTION_TIMEOUT);
+        String readTimeoutStr = System.getProperty(ND4JSystemProperties.RESOURCES_READ_TIMEOUT);
+        boolean validCTimeout = connTimeoutStr != null && connTimeoutStr.matches("\\d+");
+        boolean validRTimeout = readTimeoutStr != null && readTimeoutStr.matches("\\d+");
+
+        int connectTimeout = validCTimeout ? Integer.parseInt(connTimeoutStr) : DEFAULT_CONNECTION_TIMEOUT;
+        int readTimeout = validRTimeout ? Integer.parseInt(readTimeoutStr) : DEFAULT_READ_TIMEOUT;
+
+        try {
+            boolean correctHash = false;
+            for (int tryCount = 0; tryCount < MAX_DOWNLOAD_ATTEMPTS; tryCount++) {
+                try {
+                    if (tempFile.exists())
+                        tempFile.delete();
+                    log.info("Downloading remote resource {} to {}", remotePath, tempFile);
+                    FileUtils.copyURLToFile(new URL(remotePath), tempFile, connectTimeout, readTimeout);
+                    //Now: check if downloaded archive hash is OK
+                    String hash = sha256(tempFile);
+                    correctHash = sha256Compressed.equals(hash);
+                    if (!correctHash) {
+                        log.warn("Download of file {} failed: expected hash {} vs. actual hash {}", remotePath, sha256Compressed, hash);
+                        continue;
+                    }
+                    log.info("Downloaded {} to temporary file {}", remotePath, tempFile);
+                    break;
+                } catch (Throwable t) {
+                    if (tryCount == MAX_DOWNLOAD_ATTEMPTS - 1) {
+                        throw new RuntimeException("Error downloading test resource: " + remotePath, t);
+                    }
+                    log.warn("Error downloading test resource, retrying... {}", remotePath, t);
+                }
+            }
+
+            if (!correctHash) {
+                throw new RuntimeException("Could not successfully download with correct hash file after " + MAX_DOWNLOAD_ATTEMPTS +
+                        " attempts: " + remotePath);
+            }
+
+            //Now, extract:
+            f.getParentFile().mkdirs();
+            try (OutputStream os = new BufferedOutputStream(new FileOutputStream(f));
+                 InputStream is = new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(tempFile)))) {
+                IOUtils.copy(is, os);
+            } catch (IOException e) {
+                throw new RuntimeException("Error extracting resource file", e);
+            }
+            log.info("Extracted {} to {}", tempFile, f);
+
+            //Check extracted file hash:
+            String extractedHash = sha256(f);
+            if (!extractedHash.equals(sha256Uncompressed)) {
+                throw new RuntimeException("Extracted file hash does not match expected hash: " + remotePath +
+                        " -> " + f.getAbsolutePath() + " - expected has " + sha256Uncompressed + ", actual hash " + extractedHash);
+            }
+
+        } finally {
+            tempFile.delete();
+        }
+
+        return f;
+    }
+
+    public static String sha256(File f) {
+        try (InputStream is = new BufferedInputStream(new FileInputStream(f))) {
+            return DigestUtils.sha256Hex(is);
+        } catch (IOException e) {
+            throw new RuntimeException("Error when hashing file: " + f.getPath(), e);
+        }
+    }
+
+
+    public static final ObjectMapper newMapper() {
+        ObjectMapper ret = new ObjectMapper();
+        ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
+        ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true);
+        ret.enable(SerializationFeature.INDENT_OUTPUT);
+        return ret;
+    }
+}

resources/src/main/java/org/nd4j/common/resources/strumpf/StrumpfResolver.java

@@ -0,0 +1,284 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.common.resources.strumpf;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.nd4j.common.config.ND4JEnvironmentVars;
+import org.nd4j.common.config.ND4JSystemProperties;
+import org.nd4j.common.io.ClassPathResource;
+import org.nd4j.common.resources.Resolver;
+
+import java.io.*;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+@Slf4j
+public class StrumpfResolver implements Resolver {
+    public static final String DEFAULT_CACHE_DIR = new File(System.getProperty("user.home"), ".cache/nd4j/test_resources").getAbsolutePath();
+    public static final String REF = ".resource_reference";
+
+    protected final List<String> localResourceDirs;
+    protected final File cacheDir;
+
+    public StrumpfResolver() {
+
+        String localDirs = System.getProperty(ND4JSystemProperties.RESOURCES_LOCAL_DIRS, null);
+
+        if (localDirs != null && !localDirs.isEmpty()) {
+            String[] split = localDirs.split(",");
+            localResourceDirs = Arrays.asList(split);
+        } else {
+            localResourceDirs = null;
+        }
+
+        String cd = System.getenv(ND4JEnvironmentVars.ND4J_RESOURCES_CACHE_DIR);
+        if(cd == null || cd.isEmpty()) {
+            cd = System.getProperty(ND4JSystemProperties.RESOURCES_CACHE_DIR, DEFAULT_CACHE_DIR);
+        }
+        cacheDir = new File(cd);
+        cacheDir.mkdirs();
+    }
+
+    public int priority() {
+        return 100;
+    }
+
+    @Override
+    public boolean exists(@NonNull String resourcePath) {
+        //First: check local dirs (if any exist)
+        if (localResourceDirs != null && !localResourceDirs.isEmpty()) {
+            for (String s : localResourceDirs) {
+                //Check for standard file:
+                File f1 = new File(s, resourcePath);
+                if (f1.exists() && f1.isFile()) {
+                    //OK - found actual file
+                    return true;
+                }
+
+                //Check for reference file:
+                File f2 = new File(s, resourcePath + REF);
+                if (f2.exists() && f2.isFile()) {
+                    //OK - found resource reference
+                    return false;
+                }
+            }
+        }
+
+        //Second: Check classpath
+        ClassPathResource cpr = new ClassPathResource(resourcePath + REF);
+        if (cpr.exists()) {
+            return true;
+        }
+
+        cpr = new ClassPathResource(resourcePath);
+        if (cpr.exists()) {
+            return true;
+        }
+
+        return false;
+    }
+
+    @Override
+    public boolean directoryExists(String dirPath) {
+        //First: check local dirs (if any)
+        if (localResourceDirs != null && !localResourceDirs.isEmpty()) {
+            for (String s : localResourceDirs) {
+                File f1 = new File(s, dirPath);
+                if (f1.exists() && f1.isDirectory()) {
+                    //OK - found directory
+                    return true;
+                }
+            }
+        }
+
+        //Second: Check classpath
+        ClassPathResource cpr = new ClassPathResource(dirPath);
+        if (cpr.exists()) {
+            return true;
+        }
+
+        return false;
+    }
+
+    @Override
+    public File asFile(String resourcePath) {
+        assertExists(resourcePath);
+
+        if (localResourceDirs != null && !localResourceDirs.isEmpty()) {
+            for (String s : localResourceDirs) {
+                File f1 = new File(s, resourcePath);
+                if (f1.exists() && f1.isFile()) {
+                    //OK - found actual file
+                    return f1;
+                }
+
+                //Check for reference file:
+                File f2 = new File(s, resourcePath + REF);
+                if (f2.exists() && f2.isFile()) {
+                    //OK - found resource reference. Need to download to local cache... and/or validate what we have in cache
+                    ResourceFile rf = ResourceFile.fromFile(s);
+                    return rf.localFile(cacheDir);
+                }
+            }
+        }
+
+
+        //Second: Check classpath for references (and actual file)
+        ClassPathResource cpr = new ClassPathResource(resourcePath + REF);
+        if (cpr.exists()) {
+            ResourceFile rf;
+            try {
+                rf = ResourceFile.fromFile(cpr.getFile());
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+            return rf.localFile(cacheDir);
+        }
+
+        cpr = new ClassPathResource(resourcePath);
+        if (cpr.exists()) {
+            try {
+                return cpr.getFile();
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        throw new RuntimeException("Could not find resource file that should exist: " + resourcePath);
+    }
+
+    @Override
+    public InputStream asStream(String resourcePath) {
+        File f = asFile(resourcePath);
+        log.debug("Resolved resource " + resourcePath + " as file at absolute path " + f.getAbsolutePath());
+        try {
+            return new BufferedInputStream(new FileInputStream(f));
+        } catch (FileNotFoundException e) {
+            throw new RuntimeException("Error reading file for resource: \"" + resourcePath + "\" resolved to \"" + f + "\"");
+        }
+    }
+
+    @Override
+    public void copyDirectory(String dirPath, File destinationDir) {
+        //First: check local resource dir
+        boolean resolved = false;
+        if (localResourceDirs != null && !localResourceDirs.isEmpty()) {
+            for (String s : localResourceDirs) {
+                File f1 = new File(s, dirPath);
+                try {
+                    FileUtils.copyDirectory(f1, destinationDir);
+                    resolved = true;
+                    break;
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        }
+
+        //Second: Check classpath
+        if (!resolved) {
+            ClassPathResource cpr = new ClassPathResource(dirPath);
+            if (cpr.exists()) {
+                try {
+                    cpr.copyDirectory(destinationDir);
+                    resolved = true;
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        }
+
+        if (!resolved) {
+            throw new RuntimeException("Unable to find resource directory for path: " + dirPath);
+        }
+
+        //Finally, scan directory (recursively) and replace any resource files with actual files...
+        final List<Path> toResolve = new ArrayList<>();
+        try {
+            Files.walkFileTree(destinationDir.toPath(), new SimpleFileVisitor<Path>() {
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                    if (file.toString().endsWith(REF)) {
+                        toResolve.add(file);
+                    }
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+        if (toResolve.size() > 0) {
+            for (Path p : toResolve) {
+                File localFile = ResourceFile.fromFile(p.toFile()).localFile(cacheDir);
+                String newPath = p.toFile().getAbsolutePath();
+                newPath = newPath.substring(0, newPath.length() - REF.length());
+                File destination = new File(newPath);
+                try {
+                    FileUtils.copyFile(localFile, destination);
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+                try {
+                    FileUtils.forceDelete(p.toFile());
+                } catch (IOException e) {
+                    throw new RuntimeException("Error deleting temporary reference file", e);
+                }
+            }
+        }
+    }
+
+    @Override
+    public boolean hasLocalCache() {
+        return true;
+    }
+
+    @Override
+    public File localCacheRoot() {
+        return cacheDir;
+    }
+
+    @Override
+    public String normalizePath(@NonNull String path) {
+        if(path.endsWith(REF)){
+            return path.substring(0, path.length()-REF.length());
+        }
+        return path;
+    }
+
+
+    protected void assertExists(String resourcePath) {
+        if (!exists(resourcePath)) {
+            throw new IllegalStateException("Could not find resource with path \"" + resourcePath + "\" in local directories (" +
+                    localResourceDirs + ") or in classpath");
+        }
+    }
+
+
+}