chore: import upstream snapshot with attribution

resources/src/main/java/org/deeplearning4j/common/resources/DL4JResources.java

@@ -0,0 +1,150 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.common.resources;
+
+import lombok.NonNull;
+import org.deeplearning4j.config.DL4JSystemProperties;
+import org.nd4j.common.base.Preconditions;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+public class DL4JResources {
+
+    /**
+     * @deprecated Use {@link DL4JSystemProperties#DL4J_RESOURCES_DIR_PROPERTY}
+     */
+    @Deprecated
+    public static final String DL4J_RESOURCES_DIR_PROPERTY = DL4JSystemProperties.DL4J_RESOURCES_DIR_PROPERTY;
+    /**
+     * @deprecated Use {@link DL4JSystemProperties#DL4J_RESOURCES_BASE_URL_PROPERTY}
+     */
+    @Deprecated
+    public static final String DL4J_BASE_URL_PROPERTY = DL4JSystemProperties.DL4J_RESOURCES_BASE_URL_PROPERTY;
+    private static final String DL4J_DEFAULT_URL = "https://dl4jdata.blob.core.windows.net/";
+
+    private static File baseDirectory;
+    private static String baseURL;
+
+    static {
+        resetBaseDirectoryLocation();
+
+        String property = System.getProperty(DL4JSystemProperties.DL4J_RESOURCES_BASE_URL_PROPERTY);
+        if(property != null){
+            baseURL = property;
+        } else {
+            baseURL = DL4J_DEFAULT_URL;
+        }
+
+    }
+
+    /**
+     * Set the base download URL for (most) DL4J datasets and models.<br>
+     * This usually doesn't need to be set manually unless there is some issue with the default location
+     * @param baseDownloadURL Base download URL to set. For example, https://dl4jdata.blob.core.windows.net/
+     */
+    public static void setBaseDownloadURL(@NonNull String baseDownloadURL){
+        baseURL = baseDownloadURL;
+    }
+
+    /**
+     * @return The base URL hosting DL4J datasets and models
+     */
+    public static String getBaseDownloadURL(){
+        return baseURL;
+    }
+
+    /**
+     * Get the URL relative to the base URL.<br>
+     * For example, if baseURL is "https://dl4jdata.blob.core.windows.net/", and relativeToBase is "/datasets/iris.dat"
+     * this simply returns "https://dl4jdata.blob.core.windows.net/datasets/iris.dat"
+     *
+     * @param relativeToBase Relative URL
+     * @return URL
+     * @throws MalformedURLException For bad URL
+     */
+    public static URL getURL(String relativeToBase) throws MalformedURLException {
+        return new URL(getURLString(relativeToBase));
+    }
+
+    /**
+     * Get the URL relative to the base URL as a String.<br>
+     * For example, if baseURL is "https://dl4jdata.blob.core.windows.net/", and relativeToBase is "/datasets/iris.dat"
+     * this simply returns "https://dl4jdata.blob.core.windows.net/datasets/iris.dat"
+     *
+     * @param relativeToBase Relative URL
+     * @return URL
+     * @throws MalformedURLException For bad URL
+     */
+    public static String getURLString(String relativeToBase) {
+        if(relativeToBase.startsWith("/")){
+            relativeToBase = relativeToBase.substring(1);
+        }
+        return baseURL + relativeToBase;
+    }
+
+    /**
+     * Reset to the default directory, or the directory set via the {@link DL4JSystemProperties#DL4J_RESOURCES_DIR_PROPERTY} system property,
+     * org.deeplearning4j.resources.directory
+     */
+    public static void resetBaseDirectoryLocation(){
+        String property = System.getProperty(DL4JSystemProperties.DL4J_RESOURCES_DIR_PROPERTY);
+        if(property != null){
+            baseDirectory = new File(property);
+        } else {
+            baseDirectory = new File(System.getProperty("user.home"), ".deeplearning4j");
+        }
+
+        if(!baseDirectory.exists()){
+            baseDirectory.mkdirs();
+        }
+    }
+
+    /**
+     * Set the base directory for local storage of files. Default is: {@code new File(System.getProperty("user.home"), ".deeplearning4j")}
+     * @param f Base directory to use for resources
+     */
+    public static void setBaseDirectory(@NonNull File f){
+        Preconditions.checkState(f.exists() && f.isDirectory(), "Specified base directory does not exist and/or is not a directory: %s", f.getAbsolutePath());
+        baseDirectory = f;
+    }
+
+    /**
+     * @return The base storage directory for DL4J resources
+     */
+    public static File getBaseDirectory(){
+        return baseDirectory;
+    }
+
+    /**
+     * Get the storage location for the specified resource type and resource name
+     * @param resourceType Type of resource
+     * @param resourceName Name of the resource
+     * @return The root directory. Creates the directory and any parent directories, if required
+     */
+    public static File getDirectory(ResourceType resourceType, String resourceName){
+        File f = new File(baseDirectory, resourceType.resourceName());
+        f = new File(f, resourceName);
+        f.mkdirs();
+        return f;
+    }
+}

resources/src/main/java/org/deeplearning4j/common/resources/ResourceType.java

@@ -0,0 +1,42 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.common.resources;
+
+public enum ResourceType {
+
+    DATASET,
+    ZOO_MODEL,
+    RESOURCE;
+
+    public String resourceName(){
+        switch (this){
+            case DATASET:
+                return "data";
+            case ZOO_MODEL:
+                return "models";
+            case RESOURCE:
+                return "resources";
+            default:
+                return this.toString();
+        }
+    }
+
+}

resources/src/main/java/org/deeplearning4j/common/util/ND4JFileUtils.java

@@ -0,0 +1,52 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.common.util;
+
+import org.deeplearning4j.config.DL4JSystemProperties;
+
+import java.io.File;
+import java.io.IOException;
+
+public class ND4JFileUtils {
+
+    private ND4JFileUtils(){ }
+
+    /**
+     * Create a temporary file in the location specified by {@link DL4JSystemProperties#DL4J_TEMP_DIR_PROPERTY} if set,
+     * or the default temporary directory (usually specified by java.io.tmpdir system property)
+     * @param prefix Prefix for generating file's name; must be at least 3 characeters
+     * @param suffix Suffix for generating file's name; may be null (".tmp" will be used if null)
+     * @return A temporary file
+     */
+    public static File createTempFile(String prefix, String suffix) {
+        String p = System.getProperty(DL4JSystemProperties.DL4J_TEMP_DIR_PROPERTY);
+        try {
+            if (p == null || p.isEmpty()) {
+                return File.createTempFile(prefix, suffix);
+            } else {
+                return File.createTempFile(prefix, suffix, new File(p));
+            }
+        } catch (IOException e){
+            throw new RuntimeException("Error creating temporary file", e);
+        }
+    }
+
+}

resources/src/main/java/org/deeplearning4j/config/DL4JClassLoading.java

@@ -0,0 +1,124 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.config;
+
+import lombok.extern.slf4j.Slf4j;
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.common.config.ND4JClassLoading;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.Objects;
+import java.util.ServiceLoader;
+
+@Slf4j
+public class DL4JClassLoading {
+    private static ClassLoader dl4jClassloader = ND4JClassLoading.getNd4jClassloader();
+
+    private DL4JClassLoading() {
+    }
+
+    public static ClassLoader getDl4jClassloader() {
+        return DL4JClassLoading.dl4jClassloader;
+    }
+
+    public static void setDl4jClassloaderFromClass(Class<?> clazz) {
+        setDl4jClassloader(clazz.getClassLoader());
+    }
+
+    public static void setDl4jClassloader(ClassLoader dl4jClassloader) {
+        DL4JClassLoading.dl4jClassloader = dl4jClassloader;
+        log.debug("Global class-loader for DL4J was changed.");
+    }
+
+    public static boolean classPresentOnClasspath(String className) {
+        return classPresentOnClasspath(className, dl4jClassloader);
+    }
+
+    public static boolean classPresentOnClasspath(String className, ClassLoader classLoader) {
+        return loadClassByName(className, false, classLoader) != null;
+    }
+
+    public static <T> Class<T> loadClassByName(String className) {
+        return loadClassByName(className, true, dl4jClassloader);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static <T> Class<T> loadClassByName(String className, boolean initialize, ClassLoader classLoader) {
+        try {
+            return (Class<T>) Class.forName(className, initialize, classLoader);
+        } catch (ClassNotFoundException classNotFoundException) {
+            log.error(String.format("Cannot find class [%s] of provided class-loader.", className));
+            return null;
+        }
+    }
+
+    public static <T> T createNewInstance(String className) {
+        return createNewInstance(className, Object.class, new Object[0]);//or null;
+    }
+    
+    public static <T> T createNewInstance(String className, Object[] args) {
+       
+    	return createNewInstance(className, Object.class, args);
+    }
+
+    public static <T> T createNewInstance(String className, Class<? super T> superclass) {
+        return createNewInstance(className, superclass, new Class<?>[]{}, new Object[]{});
+    }
+
+    public static <T> T createNewInstance(String className, Class<? super T> superclass, Object[] args) {
+        Class<?>[] parameterTypes = new Class<?>[args.length];
+        for (int i = 0; i < args.length; i++) {
+            Object arg = args[i];
+            Objects.requireNonNull(arg);
+            parameterTypes[i] = arg.getClass();
+        }
+
+        return createNewInstance(className, superclass, parameterTypes, args);
+    }
+
+    public static <T> T createNewInstance(
+            String className,
+            Class<? super T> superclass,
+            Class<?>[] parameterTypes,
+            Object [] args) {
+        try {
+            Class<Object> loadedClass =  DL4JClassLoading
+                    .loadClassByName(className);
+            Preconditions.checkNotNull(loadedClass,"Attempted to load class " + className + " but failed. No class found with this name.");
+            return (T) loadedClass
+                    .asSubclass(superclass)
+                    .getDeclaredConstructor(parameterTypes)
+                    .newInstance(args);
+        } catch (InstantiationException | IllegalAccessException | InvocationTargetException
+                | NoSuchMethodException instantiationException) {
+            log.error(String.format("Cannot create instance of class '%s'.", className), instantiationException);
+            throw new RuntimeException(instantiationException);
+        }
+    }
+
+    public static <S> ServiceLoader<S> loadService(Class<S> serviceClass) {
+        return loadService(serviceClass, dl4jClassloader);
+    }
+
+    public static <S> ServiceLoader<S> loadService(Class<S> serviceClass, ClassLoader classLoader) {
+        return ServiceLoader.load(serviceClass, classLoader);
+    }
+}

resources/src/main/java/org/deeplearning4j/config/DL4JEnvironmentVars.java

@@ -0,0 +1,35 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.config;
+
+public class DL4JEnvironmentVars {
+
+    private DL4JEnvironmentVars(){ }
+
+
+    /**
+     * Applicability: Module dl4j-spark-parameterserver_2.xx<br>
+     * Usage: A fallback for determining the local IP for a Spark training worker, if other approaches
+     * fail to determine the local IP
+     */
+    public static final String DL4J_VOID_IP = "DL4J_VOID_IP";
+
+}

resources/src/main/java/org/deeplearning4j/config/DL4JSystemProperties.java

@@ -0,0 +1,130 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.config;
+
+public class DL4JSystemProperties {
+
+    private DL4JSystemProperties(){ }
+
+    /**
+     * Applicability: DL4J ModelSerializer, ModelGuesser, Keras model import<br>
+     * Description: Specify the local directory where temporary files will be written. If not specified, the default
+     * Java temporary directory (java.io.tmpdir system property) will generally be used.
+     */
+    public static final String DL4J_TEMP_DIR_PROPERTY = "org.deeplearning4j.tempdir";
+
+    /**
+     * Applicability: Numerous modules, including deeplearning4j-datasets and deeplearning4j-zoo<br>
+     * Description: Used to set the local location for downloaded remote resources such as datasets (like MNIST) and
+     * pretrained models in the model zoo. Default value is set via {@code new File(System.getProperty("user.home"), ".deeplearning4j")}.
+     * Setting this can be useful if the system drive has limited space/performance, a shared location for all users
+     * should be used instead, or if user.home isn't set for some reason.
+     */
+    public static final String DL4J_RESOURCES_DIR_PROPERTY = "org.deeplearning4j.resources.directory";
+
+    /**
+     * Applicability: Numerous modules, including deeplearning4j-datasets and deeplearning4j-zoo<br>
+     * Description: Used to set the base URL for hosting of resources such as datasets (like MNIST) and pretrained
+     * models in the model zoo. This is provided as a fallback in case the location of these files changes; it
+     * also allows for (in principle) a local mirror of these files.<br>
+     * NOTE: Changing this to a location without the same files and file structure as the DL4J resource hosting is likely
+     * to break external resource dowloading in DL4J!
+     */
+    public static final String DL4J_RESOURCES_BASE_URL_PROPERTY = "org.deeplearning4j.resources.baseurl";
+
+    /**
+     * Applicability: deeplearning4j-nn<br>
+     * Description: DL4J writes some crash dumps to disk when an OOM exception occurs - this functionality is enabled
+     * by default. This is to help users identify the cause of the OOM - i.e., where native memory is actually consumed.
+     * This system property can be used to disable memory crash reporting.
+     * @see #CRASH_DUMP_OUTPUT_DIRECTORY_PROPERTY For configuring the output directory
+     */
+    public static final String CRASH_DUMP_ENABLED_PROPERTY = "org.deeplearning4j.crash.reporting.enabled";
+
+    /**
+     * Applicability: deeplearning4j-nn<br>
+     * Description: DL4J writes some crash dumps to disk when an OOM exception occurs - this functionality is enabled
+     * by default. This system property can be use to customize the output directory for memory crash reporting. By default,
+     * the current working directory ({@code  System.getProperty("user.dir")} or {@code new File("")}) will be used
+     * @see #CRASH_DUMP_ENABLED_PROPERTY To disable crash dump reporting
+     */
+    public static final String CRASH_DUMP_OUTPUT_DIRECTORY_PROPERTY = "org.deeplearning4j.crash.reporting.directory";
+
+    /**
+     * Applicability: deeplearning4j-ui<br>
+     * Description: The DL4J training UI (StatsListener + UIServer.getInstance().attach(ss)) will subsample the number
+     * of chart points when a lot of data is present - i.e., only a maximum number of points will be shown on each chart.
+     * This is to reduce the UI bandwidth requirements and client-side rendering cost.
+     * To increase the number of points in charts, set this property to a larger value. Default: 512 values
+     */
+    public static final String CHART_MAX_POINTS_PROPERTY = "org.deeplearning4j.ui.maxChartPoints";
+
+
+    /**
+     * Applicability: deeplearning4j-vertx (deeplearning4j-ui)<br>
+     * Description: This property sets the port that the UI will be available on. Default port: 9000.
+     * Set to 0 for a random port.
+     */
+    public static final String UI_SERVER_PORT_PROPERTY = "org.deeplearning4j.ui.port";
+
+    /**
+     * Applicability: dl4j-spark_2.xx - NTPTimeSource class (mainly used in ParameterAveragingTrainingMaster when stats
+     * collection is enabled; not enabled by default)<br>
+     * Description: This sets the NTP (network time protocol) server to be used when collecting stats. Default: 0.pool.ntp.org
+     */
+    public static final String NTP_SOURCE_SERVER_PROPERTY = "org.deeplearning4j.spark.time.NTPTimeSource.server";
+
+    /**
+     * Applicability: dl4j-spark_2.xx - NTPTimeSource class (mainly used in ParameterAveragingTrainingMaster when stats
+     * collection is enabled; not enabled by default)<br>
+     * Description: This sets the NTP (network time protocol) update frequency in milliseconds. Default: 1800000 (30 minutes)
+     */
+    public static final String NTP_SOURCE_UPDATE_FREQUENCY_MS_PROPERTY = "org.deeplearning4j.spark.time.NTPTimeSource.frequencyms";
+
+    /**
+     * Applicability: dl4j-spark_2.xx - mainly used in ParameterAveragingTrainingMaster when stats collection is enabled;
+     * not enabled by default<br>
+     * Description: This sets the time source to use for spark stats. Default: {@code org.deeplearning4j.spark.time.NTPTimeSource}
+     */
+    public static final String TIMESOURCE_CLASSNAME_PROPERTY = "org.deeplearning4j.spark.time.TimeSource";
+
+
+    /**
+     * Applicability: {@code org.deeplearning4j.nn.layers.HelperUtils}
+     * Used in whether to disable the helpers or not.
+     */
+    public static final String DISABLE_HELPER_PROPERTY = "org.eclipse.deeplearning4j.helpers.disable";
+    public final static String HELPER_DISABLE_DEFAULT_VALUE = "true";
+
+    /**
+     * Applicability: NLP SkipGram and CBOW
+     * modules controlling the max cache size
+     * for on heap parameters such as learning rates and random values.
+     */
+    public static final String NLP_CACHE_SIZE = "org.eclipse.deeplearning4j.nlp.cachesize";
+
+    /**
+     * Applicability: NLP SkipGram and CBOW
+     * modules controlling the max queue size
+     * for each cache value.
+     */
+    public static final String NLP_QUEUE_SIZE = "org.eclipse.deeplearning4j.nlp.queuesize";
+}

resources/src/main/java/org/deeplearning4j/omnihub/OmnihubConfig.java

@@ -0,0 +1,56 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.deeplearning4j.omnihub;
+
+import java.io.File;
+
+public class OmnihubConfig {
+
+    public final static String OMNIHUB_HOME = "OMNIHUB_HOME";
+    public final static String OMNIHUB_URL = "OMNIHUB_URL";
+    public static final String DEFAULT_OMNIHUB_URL = "https://raw.githubusercontent.com/KonduitAI/omnihub-zoo/main";
+
+    /**
+     * Return the omnihub hurl defaulting to
+     * {@link #DEFAULT_OMNIHUB_URL}
+     * if the {@link #OMNIHUB_URL} is not specified.
+     * @return
+     */
+    public static String getOmnihubUrl() {
+        if(System.getenv(OMNIHUB_URL) != null) {
+            return System.getenv(OMNIHUB_URL);
+        } else {
+            return DEFAULT_OMNIHUB_URL;
+        }
+    }
+
+    /**
+     * return the default omnihub home at $USER/.omnihub or
+     * value of the environment variable {@link #OMNIHUB_HOME} if applicable
+     * @return
+     */
+    public static File getOmnihubHome() {
+        if(System.getenv(OMNIHUB_HOME) != null) {
+            return new File(OMNIHUB_HOME);
+        } else {
+            return new File(System.getProperty("user.home"),".omnihub");
+        }
+    }
+}