wehub/deeplearning4j--deeplearning4j
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:47:05 +08:00
commit 4f3b7da785
7394 changed files with 2005594 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deeplearning4j/.codeclimate.yml
+32
View File
@@ -0,0 +1,32 @@
version: "2" # required to adjust maintainability checks
checks:
argument-count:
config:
threshold: 4
complex-logic:
config:
threshold: 4
file-lines:
config:
threshold: 5000
method-complexity:
config:
threshold: 10
method-count:
config:
threshold: 500
method-lines:
config:
threshold: 50
nested-control-flow:
config:
threshold: 4
return-statements:
config:
threshold: 20
similar-code:
config:
threshold: # language-specific defaults. an override will affect all languages.
identical-code:
config:
threshold: # language-specific defaults. an override will affect all languages.
deeplearning4j/buildmultiplescalaversions.sh
Executable
+83
View File
@@ -0,0 +1,83 @@
#! /bin/bash
#
# /* ******************************************************************************
# *
# *
# * This program and the accompanying materials are made available under the
# * terms of the Apache License, Version 2.0 which is available at
# * https://www.apache.org/licenses/LICENSE-2.0.
# *
# * See the NOTICE file distributed with this work for additional
# * information regarding copyright ownership.
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# * License for the specific language governing permissions and limitations
# * under the License.
# *
# * SPDX-License-Identifier: Apache-2.0
# ******************************************************************************/
#
BASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
function echoError() {
(>&2 echo "$1")
}
function sparkError() {
echoError "Changing Spark major version to 2 in the build did not change the state of your working copy, is Spark 1.x still the default ?"
exit 2
}
function scalaError() {
echoError "Changing Scala major version to 2.10 in the build did not change the state of your working copy, is Scala 2.11 still the default ?"
exit 2
}
function whatchanged() {
cd "$BASEDIR"
for i in $(git status -s --porcelain -- $(find ./ -mindepth 2 -name pom.xml)|awk '{print $2}'); do
echo "$(dirname $i)"
cd "$BASEDIR"
done
}
function beep(){ echo -e '\a';}
set -eu
./change-scala-versions.sh 2.11
./change-spark-versions.sh 1 # should be idempotent, this is the default
mvn "$@"
./change-spark-versions.sh 2
if [ -z "$(whatchanged)" ]; then
sparkError;
else
if [[ "${@#-pl}" = "$@" ]]; then
mvn -Dmaven.clean.skip=true -pl $(whatchanged| tr '\n' ',') -amd "$@"
else
# the arguments already tweak the project list ! don't tweak them more
# as this can lead to conflicts (excluding a project that's not part of
# the reactor)
mvn "$@"
fi
fi
./change-scala-versions.sh 2.10
./change-spark-versions.sh 1
if [ -z "$(whatchanged)" ]; then
scalaError;
else
if [[ "${@#-pl}" = "$@" ]]; then
mvn -Dmaven.clean.skip=true -pl $(whatchanged| tr '\n' ',') -amd "$@"
else
# the arguments already tweak the project list ! don't tweak them more
# as this can lead to conflicts (excluding a project that's not part of
# the reactor)
mvn "$@"
fi
fi
./change-scala-versions.sh 2.11 # back to the default
# notify user the build is done
beep
sleep 1; beep
sleep 1; beep
deeplearning4j/deeplearning4j-common-tests/pom.xml
+88
View File
@@ -0,0 +1,88 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ /* ******************************************************************************
~ *
~ *
~ * This program and the accompanying materials are made available under the
~ * terms of the Apache License, Version 2.0 which is available at
~ * https://www.apache.org/licenses/LICENSE-2.0.
~ *
~ * See the NOTICE file distributed with this work for additional
~ * information regarding copyright ownership.
~ * Unless required by applicable law or agreed to in writing, software
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
~ * License for the specific language governing permissions and limitations
~ * under the License.
~ *
~ * SPDX-License-Identifier: Apache-2.0
~ ******************************************************************************/
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>deeplearning4j-parent</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>deeplearning4j-common-tests</artifactId>
<properties>
<module.name>deeplearning4j.common.tests</module.name>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.moditect</groupId>
<artifactId>moditect-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-launcher</artifactId>
<version>${junit.platform.launcher.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>${junit.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>${junit.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>nd4j-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>deeplearning4j-nn</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>
deeplearning4j/deeplearning4j-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java
+98
View File
@@ -0,0 +1,98 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j;
import org.deeplearning4j.nn.conf.ConfClassLoading;
import org.junit.jupiter.api.DisplayName;
import org.nd4j.common.tools.ClassInitializerUtil;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ops.executioner.OpExecutioner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
@DisplayName("Base DL 4 J Test")
public abstract class BaseDL4JTest {
protected long startTime;
protected int threadCountBefore;
private final int DEFAULT_THREADS = Runtime.getRuntime().availableProcessors();
/**
* Override this to specify the number of threads for C++ execution, via
* {@link org.nd4j.linalg.factory.Environment#setMaxMasterThreads(int)}
* @return Number of threads to use for C++ op execution
*/
public int numThreads() {
return DEFAULT_THREADS;
}
/**
* Override this method to set the default timeout for methods in the test class
*/
public long getTimeoutMilliseconds() {
return 90_000;
}
/**
* Override this to set the profiling mode for the tests defined in the child class
*/
public OpExecutioner.ProfilingMode getProfilingMode() {
return OpExecutioner.ProfilingMode.SCOPE_PANIC;
}
/**
* Override this to set the datatype of the tests defined in the child class
*/
public DataType getDataType() {
return DataType.DOUBLE;
}
public DataType getDefaultFPDataType() {
return getDataType();
}
protected static Boolean integrationTest;
/**
* @return True if integration tests maven profile is enabled, false otherwise.
*/
public static boolean isIntegrationTests() {
if (integrationTest == null) {
String prop = System.getenv("DL4J_INTEGRATION_TESTS");
integrationTest = Boolean.parseBoolean(prop);
}
return integrationTest;
}
/**
* Call this as the first line of a test in order to skip that test, only when the integration tests maven profile is not enabled.
* This can be used to dynamically skip integration tests when the integration test profile is not enabled.
* Note that the integration test profile is not enabled by default - "integration-tests" profile
*/
public static void skipUnlessIntegrationTests() {
assumeTrue(isIntegrationTests(), "Skipping integration test - integration profile is not enabled");
}
}
deeplearning4j/deeplearning4j-common-tests/src/main/java9/module-info.java
+10
View File
@@ -0,0 +1,10 @@
open module deeplearning4j.common.tests {
requires java.management;
requires lombok;
requires nd4j.common;
requires org.bytedeco.javacpp;
requires org.junit.jupiter.api;
requires slf4j.api;
requires nd4j.api;
exports org.deeplearning4j;
}
deeplearning4j/deeplearning4j-core/pom.xml
+185
View File
@@ -0,0 +1,185 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ /* ******************************************************************************
~ *
~ *
~ * This program and the accompanying materials are made available under the
~ * terms of the Apache License, Version 2.0 which is available at
~ * https://www.apache.org/licenses/LICENSE-2.0.
~ *
~ * See the NOTICE file distributed with this work for additional
~ * information regarding copyright ownership.
~ * Unless required by applicable law or agreed to in writing, software
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
~ * License for the specific language governing permissions and limitations
~ * under the License.
~ *
~ * SPDX-License-Identifier: Apache-2.0
~ ******************************************************************************/
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>deeplearning4j-parent</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>deeplearning4j-core</artifactId>
<properties>
<test.offheap.size>14g</test.offheap.size>
<test.heap.size>14g</test.heap.size>
<module.name>deeplearning4j.core</module.name>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>nd4j-api</artifactId>
<version>${nd4j.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>nd4j-common</artifactId>
<version>${nd4j.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.moditect</groupId>
<artifactId>moditect-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
<dependencies>
<!-- Java 9 + module support needed. activation moved out of JDK-->
<dependency>
<groupId>com.sun.activation</groupId>
<artifactId>javax.activation</artifactId>
<version>1.2.0</version>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>deeplearning4j-datasets</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>deeplearning4j-datavec-iterators</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>deeplearning4j-modelimport</artifactId>
<version>${project.version}</version>
</dependency>
<!-- Logging Dependencies -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<!-- Redirect jackson to slf4j. -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>deeplearning4j-nn</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${commonsio.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>${commons-compress.version}</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-launcher</artifactId>
<version>${junit.platform.launcher.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>deeplearning4j-common-tests</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>nd4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commonslang.version}</version>
</dependency>
<!-- ND4J Shaded Jackson Dependency -->
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>jackson</artifactId>
<version>${nd4j.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>datavec-api</artifactId>
<version>${datavec.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>datavec-data-image</artifactId>
<version>${datavec.version}</version>
</dependency>
<!-- deeplearning4j-ui-components: Used for generating HTML for export (ROC curves and the like) -->
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>deeplearning4j-ui-components</artifactId>
<version>${project.version}</version>
</dependency>
<!-- Dependencies for dl4j-perf subproject -->
<dependency>
<groupId>com.github.oshi</groupId>
<artifactId>oshi-json</artifactId>
<version>${oshi.version}</version>
</dependency>
<dependency>
<groupId>com.github.oshi</groupId>
<artifactId>oshi-core</artifactId>
<version>${oshi.version}</version>
</dependency>
</dependencies>
</project>
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/datasets/test/TestDataSetIterator.java
+117
View File
@@ -0,0 +1,117 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.datasets.test;
import lombok.Getter;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import java.util.List;
public class TestDataSetIterator implements DataSetIterator {
/**
*
*/
private static final long serialVersionUID = -3042802726018263331L;
private DataSetIterator wrapped;
private int numDataSets = 0;
@Getter
private DataSetPreProcessor preProcessor;
public TestDataSetIterator(DataSetIterator wrapped) {
super();
this.wrapped = wrapped;
}
@Override
public boolean hasNext() {
return wrapped.hasNext();
}
@Override
public DataSet next() {
numDataSets++;
DataSet next = wrapped.next();
if (preProcessor != null)
preProcessor.preProcess(next);
return next;
}
@Override
public void remove() {
wrapped.remove();
}
@Override
public int inputColumns() {
return wrapped.inputColumns();
}
@Override
public int totalOutcomes() {
return wrapped.totalOutcomes();
}
@Override
public boolean resetSupported() {
return wrapped.resetSupported();
}
@Override
public boolean asyncSupported() {
return wrapped.asyncSupported();
}
@Override
public void reset() {
wrapped.reset();
}
@Override
public int batch() {
return wrapped.batch();
}
@Override
public void setPreProcessor(DataSetPreProcessor preProcessor) {
this.preProcessor = preProcessor;
}
@Override
public List<String> getLabels() {
return null;
}
public synchronized int getNumDataSets() {
return numDataSets;
}
@Override
public DataSet next(int num) {
return wrapped.next(num);
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/datasets/vectorizer/Vectorizer.java
+38
View File
@@ -0,0 +1,38 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.datasets.vectorizer;
import org.nd4j.linalg.dataset.DataSet;
import java.io.Serializable;
public interface Vectorizer extends Serializable {
/**
* Vectorizes the input source in to a dataset
* @return Adam Gibson
*/
DataSet vectorize();
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/evaluation/EvaluationTools.java
+345
View File
@@ -0,0 +1,345 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.evaluation;
import org.apache.commons.io.FileUtils;
import org.deeplearning4j.ui.api.Component;
import org.deeplearning4j.ui.api.LengthUnit;
import org.deeplearning4j.ui.components.chart.ChartHistogram;
import org.deeplearning4j.ui.components.chart.ChartLine;
import org.deeplearning4j.ui.components.chart.style.StyleChart;
import org.deeplearning4j.ui.components.component.ComponentDiv;
import org.deeplearning4j.ui.components.component.style.StyleDiv;
import org.deeplearning4j.ui.components.table.ComponentTable;
import org.deeplearning4j.ui.components.table.style.StyleTable;
import org.deeplearning4j.ui.components.text.ComponentText;
import org.deeplearning4j.ui.components.text.style.StyleText;
import org.deeplearning4j.ui.standalone.StaticPageUtil;
import org.nd4j.evaluation.classification.EvaluationCalibration;
import org.nd4j.evaluation.classification.ROC;
import org.nd4j.evaluation.classification.ROCMultiClass;
import org.nd4j.evaluation.curves.Histogram;
import org.nd4j.evaluation.curves.PrecisionRecallCurve;
import org.nd4j.evaluation.curves.ReliabilityDiagram;
import org.nd4j.evaluation.curves.RocCurve;
import java.awt.*;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
public class EvaluationTools {
private static final String ROC_TITLE = "ROC: TPR/Recall (y) vs. FPR (x)";
private static final String PR_TITLE = "Precision (y) vs. Recall (x)";
private static final String PR_THRESHOLD_TITLE = "Precision and Recall (y) vs. Classifier Threshold (x)";
private static final double CHART_WIDTH_PX = 600.0;
private static final double CHART_HEIGHT_PX = 400.0;
private static final StyleChart CHART_STYLE = new StyleChart.Builder().width(CHART_WIDTH_PX, LengthUnit.Px)
.height(CHART_HEIGHT_PX, LengthUnit.Px).margin(LengthUnit.Px, 60, 60, 75, 10).strokeWidth(2.0)
.seriesColors(Color.BLUE, Color.LIGHT_GRAY).build();
private static final StyleChart CHART_STYLE_PRECISION_RECALL =
new StyleChart.Builder().width(CHART_WIDTH_PX, LengthUnit.Px).height(CHART_HEIGHT_PX, LengthUnit.Px)
.margin(LengthUnit.Px, 60, 60, 40, 10).strokeWidth(2.0)
.seriesColors(Color.BLUE, Color.GREEN).build();
private static final StyleTable TABLE_STYLE = new StyleTable.Builder().backgroundColor(Color.WHITE)
.headerColor(Color.LIGHT_GRAY).borderWidth(1).columnWidths(LengthUnit.Percent, 50, 50)
.width(400, LengthUnit.Px).height(200, LengthUnit.Px).build();
private static final StyleDiv OUTER_DIV_STYLE = new StyleDiv.Builder().width(2 * CHART_WIDTH_PX, LengthUnit.Px)
.height(CHART_HEIGHT_PX, LengthUnit.Px).build();
private static final StyleDiv OUTER_DIV_STYLE_WIDTH_ONLY =
new StyleDiv.Builder().width(2 * CHART_WIDTH_PX, LengthUnit.Px).build();
private static final StyleDiv INNER_DIV_STYLE = new StyleDiv.Builder().width(CHART_WIDTH_PX, LengthUnit.Px)
.floatValue(StyleDiv.FloatValue.left).build();
private static final StyleDiv PAD_DIV_STYLE = new StyleDiv.Builder().width(CHART_WIDTH_PX, LengthUnit.Px)
.height(100, LengthUnit.Px).floatValue(StyleDiv.FloatValue.left).build();
private static final ComponentDiv PAD_DIV = new ComponentDiv(PAD_DIV_STYLE);
private static final StyleText HEADER_TEXT_STYLE =
new StyleText.Builder().color(Color.BLACK).fontSize(16).underline(true).build();
private static final StyleDiv HEADER_DIV_STYLE =
new StyleDiv.Builder().width(2 * CHART_WIDTH_PX - 150, LengthUnit.Px).height(30, LengthUnit.Px)
.backgroundColor(Color.LIGHT_GRAY).margin(LengthUnit.Px, 5, 5, 200, 10)
.floatValue(StyleDiv.FloatValue.left).build();
private static final StyleDiv HEADER_DIV_STYLE_1400 = new StyleDiv.Builder().width(1400 - 150, LengthUnit.Px)
.height(30, LengthUnit.Px).backgroundColor(Color.LIGHT_GRAY).margin(LengthUnit.Px, 5, 5, 200, 10)
.floatValue(StyleDiv.FloatValue.left).build();
private static final StyleDiv HEADER_DIV_PAD_STYLE = new StyleDiv.Builder().width(2 * CHART_WIDTH_PX, LengthUnit.Px)
.height(150, LengthUnit.Px).backgroundColor(Color.WHITE).build();
private static final StyleDiv HEADER_DIV_TEXT_PAD_STYLE =
new StyleDiv.Builder().width(120, LengthUnit.Px).height(30, LengthUnit.Px)
.backgroundColor(Color.LIGHT_GRAY).floatValue(StyleDiv.FloatValue.left).build();
private static final ComponentTable INFO_TABLE = new ComponentTable.Builder(
new StyleTable.Builder().backgroundColor(Color.WHITE).borderWidth(0).build())
.content(new String[][] {
{"Precision", "(true positives) / (true positives + false positives)"},
{"True Positive Rate (Recall)",
"(true positives) / (data positives)"},
{"False Positive Rate", "(false positives) / (data negatives)"}})
.build();
private EvaluationTools() {}
/**
* Given a {@link ROC} chart, export the ROC chart and precision vs. recall charts to a stand-alone HTML file
* @param roc ROC to export
* @param file File to export to
*/
public static void exportRocChartsToHtmlFile(ROC roc, File file) throws IOException {
String rocAsHtml = rocChartToHtml(roc);
FileUtils.writeStringToFile(file, rocAsHtml);
}
/**
* Given a {@link ROCMultiClass} chart, export the ROC chart and precision vs. recall charts to a stand-alone HTML file
* @param roc ROC to export
* @param file File to export to
*/
public static void exportRocChartsToHtmlFile(ROCMultiClass roc, File file) throws Exception {
String rocAsHtml = rocChartToHtml(roc);
FileUtils.writeStringToFile(file, rocAsHtml, StandardCharsets.UTF_8);
}
/**
* Given a {@link ROC} instance, render the ROC chart and precision vs. recall charts to a stand-alone HTML file (returned as a String)
* @param roc ROC to render
*/
public static String rocChartToHtml(ROC roc) {
RocCurve rocCurve = roc.getRocCurve();
Component c = getRocFromPoints(ROC_TITLE, rocCurve, roc.getCountActualPositive(), roc.getCountActualNegative(),
roc.calculateAUC(), roc.calculateAUCPR());
Component c2 = getPRCharts(PR_TITLE, PR_THRESHOLD_TITLE, roc.getPrecisionRecallCurve());
return StaticPageUtil.renderHTML(c, c2);
}
/**
* Given a {@link ROCMultiClass} instance, render the ROC chart and precision vs. recall charts to a stand-alone HTML file (returned as a String)
* @param rocMultiClass ROC to render
*/
public static String rocChartToHtml(ROCMultiClass rocMultiClass) {
return rocChartToHtml(rocMultiClass, null);
}
/**
* Given a {@link ROCMultiClass} instance and (optionally) names for each class, render the ROC chart to a stand-alone
* HTML file (returned as a String)
* @param rocMultiClass ROC to render
* @param classNames Names of the classes. May be null
*/
public static String rocChartToHtml(ROCMultiClass rocMultiClass, List<String> classNames) {
int n = rocMultiClass.getNumClasses();
List<Component> components = new ArrayList<>(n);
for (int i = 0; i < n; i++) {
RocCurve roc = rocMultiClass.getRocCurve(i);
String headerText = "Class " + i;
if (classNames != null && classNames.size() > i) {
headerText += " (" + classNames.get(i) + ")";
}
headerText += " vs. All";;
Component headerDivPad = new ComponentDiv(HEADER_DIV_PAD_STYLE);
components.add(headerDivPad);
Component headerDivLeft = new ComponentDiv(HEADER_DIV_TEXT_PAD_STYLE);
Component headerDiv = new ComponentDiv(HEADER_DIV_STYLE, new ComponentText(headerText, HEADER_TEXT_STYLE));
Component c = getRocFromPoints(ROC_TITLE, roc, rocMultiClass.getCountActualPositive(i),
rocMultiClass.getCountActualNegative(i), rocMultiClass.calculateAUC(i),
rocMultiClass.calculateAUCPR(i));
Component c2 = getPRCharts(PR_TITLE, PR_THRESHOLD_TITLE, rocMultiClass.getPrecisionRecallCurve(i));
components.add(headerDivLeft);
components.add(headerDiv);
components.add(c);
components.add(c2);
}
return StaticPageUtil.renderHTML(components);
}
/**
* Given a {@link EvaluationCalibration} instance, export the charts to a stand-alone HTML file
* @param ec EvaluationCalibration instance to export HTML charts for
* @param file File to export to
*/
public static void exportevaluationCalibrationToHtmlFile(EvaluationCalibration ec, File file) throws IOException {
String asHtml = evaluationCalibrationToHtml(ec);
FileUtils.writeStringToFile(file, asHtml);
}
public static String evaluationCalibrationToHtml(EvaluationCalibration ec) {
List<Component> components = new ArrayList<>();
int nClasses = ec.numClasses();
//Distribution of class labels + distribution of predicted classes
Component headerDiv = new ComponentDiv(HEADER_DIV_STYLE_1400,
new ComponentText(
"Labels and Network Prediction Class Distributions (X: Class Index. Y: Count)",
HEADER_TEXT_STYLE));
components.add(headerDiv);
int[] labelCounts = ec.getLabelCountsEachClass();
int[] predictedCounts = ec.getPredictionCountsEachClass();
ChartHistogram.Builder chbLabels = new ChartHistogram.Builder("Label Class Distribution", CHART_STYLE);
ChartHistogram.Builder chbPredictions = new ChartHistogram.Builder("Predicted Class Distribution", CHART_STYLE);
for (int i = 0; i < nClasses; i++) {
double lower = i - 0.5;
double upper = i + 0.5;
chbLabels.addBin(lower, upper, labelCounts[i]);
chbPredictions.addBin(lower, upper, predictedCounts[i]);
}
ChartHistogram chL = chbLabels.build();
ChartHistogram chP = chbPredictions.build();
components.add(new ComponentDiv(OUTER_DIV_STYLE_WIDTH_ONLY, chL, chP));
//Reliability diagram, for each class
headerDiv = new ComponentDiv(HEADER_DIV_STYLE_1400, new ComponentText(
"Reliability Diagrams (X: Mean Predicted Value. Y: Fraction Positives)", HEADER_TEXT_STYLE));
components.add(headerDiv);
List<Component> sectionDiv = new ArrayList<>();
double[] zeroOne = new double[] {0.0, 1.0};
for (int i = 0; i < nClasses; i++) {
ReliabilityDiagram rd = ec.getReliabilityDiagram(i);
double[] x = rd.getMeanPredictedValueX();
double[] y = rd.getFractionPositivesY();
String title = rd.getTitle();
ChartLine cl = new ChartLine.Builder(title, CHART_STYLE).addSeries("Classifier", x, y)
.addSeries("Ideal Classifier", zeroOne, zeroOne).build();
sectionDiv.add(cl);
}
components.add(new ComponentDiv(OUTER_DIV_STYLE_WIDTH_ONLY, sectionDiv));
//Residual plots
headerDiv = new ComponentDiv(HEADER_DIV_STYLE_1400, new ComponentText(
"Network Predictions - Residual Plots - |Label(i) - P(class(i))|", HEADER_TEXT_STYLE));
components.add(headerDiv);
sectionDiv = new ArrayList<>();
Histogram resPlotAll = ec.getResidualPlotAllClasses();
sectionDiv.add(getHistogram(resPlotAll));
for (int i = 0; i < nClasses; i++) {
Histogram resPlotCurrent = ec.getResidualPlot(i);
sectionDiv.add(getHistogram(resPlotCurrent));
}
components.add(new ComponentDiv(OUTER_DIV_STYLE_WIDTH_ONLY, sectionDiv));
//Histogram of probabilities, overall and for each class
headerDiv = new ComponentDiv(HEADER_DIV_STYLE_1400, new ComponentText(
"Network Prediction Probabilities (X: P(class). Y: Count)", HEADER_TEXT_STYLE));
components.add(headerDiv);
sectionDiv = new ArrayList<>();
Histogram allProbs = ec.getProbabilityHistogramAllClasses();
sectionDiv.add(getHistogram(allProbs));
for (int i = 0; i < nClasses; i++) {
Histogram classProbs = ec.getProbabilityHistogram(i);
sectionDiv.add(getHistogram(classProbs));
}
components.add(new ComponentDiv(OUTER_DIV_STYLE_WIDTH_ONLY, sectionDiv));
return StaticPageUtil.renderHTML(components);
}
private static Component getRocFromPoints(String title, RocCurve roc, long positiveCount, long negativeCount,
double auc, double aucpr) {
double[] zeroOne = new double[] {0.0, 1.0};
ChartLine chartLine = new ChartLine.Builder(title, CHART_STYLE).setXMin(0.0).setXMax(1.0).setYMin(0.0)
.setYMax(1.0).addSeries("ROC", roc.getX(), roc.getY()).addSeries("", zeroOne, zeroOne).build();
ComponentTable ct = new ComponentTable.Builder(TABLE_STYLE).header("Field", "Value")
.content(new String[][] {{"AUROC: Area under ROC:", String.format("%.5f", auc)},
{"AUPRC: Area under P/R:", String.format("%.5f", aucpr)},
{"Total Data Positive Count", String.valueOf(positiveCount)},
{"Total Data Negative Count", String.valueOf(negativeCount)}})
.build();
ComponentDiv divLeft = new ComponentDiv(INNER_DIV_STYLE, PAD_DIV, ct, PAD_DIV, INFO_TABLE);
ComponentDiv divRight = new ComponentDiv(INNER_DIV_STYLE, chartLine);
return new ComponentDiv(OUTER_DIV_STYLE, divLeft, divRight);
}
private static Component getPRCharts(String precisionRecallTitle, String prThresholdTitle,
PrecisionRecallCurve prCurve) {
ComponentDiv divLeft =
new ComponentDiv(INNER_DIV_STYLE, getPrecisionRecallCurve(precisionRecallTitle, prCurve));
ComponentDiv divRight =
new ComponentDiv(INNER_DIV_STYLE, getPrecisionRecallVsThreshold(prThresholdTitle, prCurve));
return new ComponentDiv(OUTER_DIV_STYLE, divLeft, divRight);
}
private static Component getPrecisionRecallCurve(String title, PrecisionRecallCurve prCurve) {
double[] recallX = prCurve.getRecall();
double[] precisionY = prCurve.getPrecision();
return new ChartLine.Builder(title, CHART_STYLE).setXMin(0.0).setXMax(1.0).setYMin(0.0).setYMax(1.0)
.addSeries("P vs R", recallX, precisionY).build();
}
private static Component getPrecisionRecallVsThreshold(String title, PrecisionRecallCurve prCurve) {
double[] recallY = prCurve.getRecall();
double[] precisionY = prCurve.getPrecision();
double[] thresholdX = prCurve.getThreshold();
return new ChartLine.Builder(title, CHART_STYLE_PRECISION_RECALL).setXMin(0.0).setXMax(1.0).setYMin(0.0)
.setYMax(1.0).addSeries("Precision", thresholdX, precisionY)
.addSeries("Recall", thresholdX, recallY).showLegend(true).build();
}
private static Component getHistogram(Histogram histogram) {
ChartHistogram.Builder chb = new ChartHistogram.Builder(histogram.getTitle(), CHART_STYLE);
double[] lower = histogram.getBinLowerBounds();
double[] upper = histogram.getBinUpperBounds();
int[] counts = histogram.getBinCounts();
for (int i = 0; i < counts.length; i++) {
chb.addBin(lower[i], upper[i], counts[i]);
}
return chb.build();
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/DeviceMetric.java
+45
View File
@@ -0,0 +1,45 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.listener;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import java.io.Serializable;
@Data
@Builder
@AllArgsConstructor
public class DeviceMetric implements Serializable {
private double load;
private double totalMemory;
private String deviceName;
private double temp;
private double memAvailable;
private long bandwidthDeviceToHost,bandwidthHostToDevice,bandwidthDeviceToDevice;
private DeviceMetric(){
//No-arg constructor for JSON/YAML
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/DiskInfo.java
+39
View File
@@ -0,0 +1,39 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.listener;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import java.io.Serializable;
@Data
@Builder
@AllArgsConstructor
public class DiskInfo implements Serializable {
private long bytesRead,bytesWritten,transferTime;
private String name,modelName;
private DiskInfo(){
//No-arg for JSON/YAML
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/HardwareMetric.java
+187
View File
@@ -0,0 +1,187 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.listener;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NonNull;
import org.nd4j.linalg.api.environment.Nd4jEnvironment;
import org.nd4j.linalg.api.ops.performance.PerformanceTracker;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.api.memory.MemcpyDirection;
import org.nd4j.shade.jackson.databind.ObjectMapper;
import org.nd4j.shade.jackson.dataformat.yaml.YAMLFactory;
import oshi.json.SystemInfo;
import oshi.json.hardware.CentralProcessor;
import oshi.json.hardware.GlobalMemory;
import oshi.json.hardware.HWDiskStore;
import oshi.json.software.os.NetworkParams;
import oshi.util.Util;
import java.io.IOException;
import java.io.Serializable;
import java.util.*;
@Builder
@Data
@AllArgsConstructor
public class HardwareMetric implements Serializable {
private static ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory());
private Map<Integer,DeviceMetric> perCoreMetrics;
private long physicalProcessorCount,logicalProcessorCount;
private long currentMemoryUse;
private Map<Integer,DeviceMetric> gpuMetrics;
private String hostName;
private long ioWaitTime;
private long averagedCpuLoad;
private Map<Integer,DiskInfo> diskInfo;
private String name;
private HardwareMetric(){
//No-arg for JSON/YAML
}
/**
* Runs {@link #fromSystem(SystemInfo)}
* with a fresh {@link SystemInfo}
* @return the hardware metric based on
* the current snapshot of the system this
* runs on
*/
public static HardwareMetric fromSystem() {
return fromSystem(new SystemInfo());
}
/**
* Returns the relevant information
* needed for system diagnostics
* based on the {@link SystemInfo}
* @param systemInfo the system info to use
* @return the {@link HardwareMetric} for the
* system this process runs on
*/
public static HardwareMetric fromSystem(SystemInfo systemInfo) {
return fromSystem(systemInfo,UUID.randomUUID().toString());
}
/**
* Returns the relevant information
* needed for system diagnostics
* based on the {@link SystemInfo}
* @param systemInfo the system info to use
* @return the {@link HardwareMetric} for the
* system this process runs on
*/
public static HardwareMetric fromSystem(SystemInfo systemInfo,String name) {
HardwareMetricBuilder builder = HardwareMetric.builder();
CentralProcessor processor = systemInfo.getHardware().getProcessor();
long[] prevTicks = processor.getSystemCpuLoadTicks();
// Wait a second...
Util.sleep(1000);
long[] ticks = processor.getSystemCpuLoadTicks();
long iowait = ticks[oshi.hardware.CentralProcessor.TickType.IOWAIT.getIndex()] - prevTicks[oshi.hardware.CentralProcessor.TickType.IOWAIT.getIndex()];
GlobalMemory globalMemory = systemInfo.getHardware().getMemory();
NetworkParams networkParams = systemInfo.getOperatingSystem().getNetworkParams();
double[] processorCpuLoadBetweenTicks = processor.getProcessorCpuLoadBetweenTicks();
Map<Integer,DeviceMetric> cpuMetrics = new LinkedHashMap<>();
for(int i = 0; i < processorCpuLoadBetweenTicks.length; i++) {
cpuMetrics.put(i, DeviceMetric.builder()
.load(processorCpuLoadBetweenTicks[i]).
build());
}
Map<Integer,DiskInfo> diskInfoMap = new LinkedHashMap<>();
HWDiskStore[] diskStores = systemInfo.getHardware().getDiskStores();
for(int i = 0; i < diskStores.length; i++) {
HWDiskStore diskStore = diskStores[i];
DiskInfo diskInfo = DiskInfo.builder()
.bytesRead(diskStore.getReadBytes())
.bytesWritten(diskStore.getWriteBytes())
.name(diskStore.getName())
.modelName(diskStore.getModel())
.transferTime(diskStore.getTransferTime())
.build();
diskInfoMap.put(i,diskInfo);
}
Map<Integer,DeviceMetric> gpuMetric = new HashMap<>();
if(Nd4j.getBackend().getClass().getName().toLowerCase().contains("cublas")) {
Properties info = Nd4j.getExecutioner().getEnvironmentInformation();
/**
*
*/
List<Map<String, Object>> devicesList = (List<Map<String, Object>>) info.get(Nd4jEnvironment.CUDA_DEVICE_INFORMATION_KEY);
for(int i = 0; i < devicesList.size(); i++) {
double available = Double.parseDouble(devicesList.get(i).get(Nd4jEnvironment.CUDA_FREE_MEMORY_KEY).toString());
Map<MemcpyDirection, Long> memcpyDirectionLongMap = PerformanceTracker.getInstance().getCurrentBandwidth().get(i);
DeviceMetric deviceMetric = DeviceMetric.builder()
.bandwidthHostToDevice(memcpyDirectionLongMap.get(MemcpyDirection.HOST_TO_DEVICE))
.bandwidthDeviceToHost(memcpyDirectionLongMap.get(MemcpyDirection.DEVICE_TO_HOST))
.bandwidthDeviceToDevice(memcpyDirectionLongMap.get(MemcpyDirection.DEVICE_TO_DEVICE))
.memAvailable(available).totalMemory(Double.parseDouble(devicesList.get(i).get(Nd4jEnvironment.CUDA_TOTAL_MEMORY_KEY).toString()))
.deviceName(devicesList.get(i).get(Nd4jEnvironment.CUDA_DEVICE_NAME_KEY).toString())
.build();
gpuMetric.put(i,deviceMetric);
}
}
return builder.logicalProcessorCount(processor.getLogicalProcessorCount())
.physicalProcessorCount(processor.getPhysicalProcessorCount())
.name(name)
.averagedCpuLoad((long)(processor.getSystemCpuLoad() * 100))
.ioWaitTime(iowait).gpuMetrics(gpuMetric)
.hostName(networkParams.getHostName()).diskInfo(diskInfoMap)
.currentMemoryUse(globalMemory.getTotal() - globalMemory.getAvailable())
.perCoreMetrics(cpuMetrics)
.build();
}
public String toYaml(){
try {
return yamlMapper.writeValueAsString(this);
} catch (Exception e){
throw new RuntimeException(e);
}
}
public static HardwareMetric fromYaml(@NonNull String yaml){
try {
return yamlMapper.readValue(yaml, HardwareMetric.class);
} catch (IOException e){
throw new RuntimeException(e);
}
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/SystemInfoFilePrintListener.java
+132
View File
@@ -0,0 +1,132 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.listener;
import lombok.NonNull;
import lombok.Builder;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.deeplearning4j.nn.api.Model;
import org.deeplearning4j.optimize.api.TrainingListener;
import org.nd4j.linalg.api.ndarray.INDArray;
import oshi.json.SystemInfo;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
@Slf4j
@Builder
public class SystemInfoFilePrintListener implements TrainingListener {
private boolean printOnEpochStart;
private boolean printOnEpochEnd;
private boolean printOnForwardPass;
private boolean printOnBackwardPass;
private boolean printOnGradientCalculation;
private File printFileTarget;
public SystemInfoFilePrintListener(boolean printOnEpochStart, boolean printOnEpochEnd, boolean printOnForwardPass, boolean printOnBackwardPass, boolean printOnGradientCalculation, @NonNull File printFileTarget) {
this.printOnEpochStart = printOnEpochStart;
this.printOnEpochEnd = printOnEpochEnd;
this.printOnForwardPass = printOnForwardPass;
this.printOnBackwardPass = printOnBackwardPass;
this.printOnGradientCalculation = printOnGradientCalculation;
this.printFileTarget = printFileTarget;
}
@Override
public void iterationDone(Model model, int iteration, int epoch) {
}
@Override
public void onEpochStart(Model model) {
if(!printOnEpochStart || printFileTarget == null)
return;
writeFileWithMessage("epoch end");
}
@Override
public void onEpochEnd(Model model) {
if(!printOnEpochEnd || printFileTarget == null)
return;
writeFileWithMessage("epoch begin");
}
@Override
public void onForwardPass(Model model, List<INDArray> activations) {
if(!printOnBackwardPass || printFileTarget == null)
return;
writeFileWithMessage("forward pass");
}
@Override
public void onForwardPass(Model model, Map<String, INDArray> activations) {
if(!printOnForwardPass || printFileTarget == null)
return;
writeFileWithMessage("forward pass");
}
@Override
public void onGradientCalculation(Model model) {
if(!printOnGradientCalculation || printFileTarget == null)
return;
writeFileWithMessage("gradient calculation");
}
@Override
public void onBackwardPass(Model model) {
if(!printOnBackwardPass || printFileTarget == null)
return;
writeFileWithMessage("backward pass");
}
private void writeFileWithMessage(String status) {
if(printFileTarget == null) {
log.warn("File not specified for writing!");
}
SystemInfo systemInfo = new SystemInfo();
log.info("Writing system info to file on " + status + ": " + printFileTarget.getAbsolutePath());
try {
FileUtils.write(printFileTarget,systemInfo.toPrettyJSON(), true);
} catch (IOException e) {
log.error("Error writing file for system info",e);
}
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/SystemInfoPrintListener.java
+114
View File
@@ -0,0 +1,114 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.listener;
import lombok.Builder;
import lombok.extern.slf4j.Slf4j;
import org.deeplearning4j.nn.api.Model;
import org.deeplearning4j.optimize.api.TrainingListener;
import org.nd4j.linalg.api.ndarray.INDArray;
import oshi.json.SystemInfo;
import java.util.List;
import java.util.Map;
/**
* Using {@link SystemInfo} - it logs a json representation
* of system info using slf4j.
*
* @author Adam Gibson
*/
@Slf4j
@Builder
public class SystemInfoPrintListener implements TrainingListener {
private boolean printOnEpochStart;
private boolean printOnEpochEnd;
private boolean printOnForwardPass;
private boolean printOnBackwardPass;
private boolean printOnGradientCalculation;
private static final String SYSTEM_INFO = "System info on epoch end: ";
@Override
public void iterationDone(Model model, int iteration, int epoch) {
}
@Override
public void onEpochStart(Model model) {
if(!printOnEpochStart)
return;
SystemInfo systemInfo = new SystemInfo();
log.info("System info on epoch begin: ");
log.info(systemInfo.toPrettyJSON());
}
@Override
public void onEpochEnd(Model model) {
if(!printOnEpochEnd)
return;
SystemInfo systemInfo = new SystemInfo();
log.info(SYSTEM_INFO);
log.info(systemInfo.toPrettyJSON());
}
@Override
public void onForwardPass(Model model, List<INDArray> activations) {
if(!printOnBackwardPass)
return;
SystemInfo systemInfo = new SystemInfo();
log.info(SYSTEM_INFO);
log.info(systemInfo.toPrettyJSON());
}
@Override
public void onForwardPass(Model model, Map<String, INDArray> activations) {
if(!printOnForwardPass)
return;
SystemInfo systemInfo = new SystemInfo();
log.info(SYSTEM_INFO);
log.info(systemInfo.toPrettyJSON());
}
@Override
public void onGradientCalculation(Model model) {
if(!printOnGradientCalculation)
return;
SystemInfo systemInfo = new SystemInfo();
log.info(SYSTEM_INFO);
log.info(systemInfo.toPrettyJSON());
}
@Override
public void onBackwardPass(Model model) {
if(!printOnBackwardPass)
return;
SystemInfo systemInfo = new SystemInfo();
log.info(SYSTEM_INFO);
log.info(systemInfo.toPrettyJSON());
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/SystemPolling.java
+142
View File
@@ -0,0 +1,142 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.listener;
import lombok.extern.slf4j.Slf4j;
import org.nd4j.shade.jackson.databind.ObjectMapper;
import org.nd4j.shade.jackson.dataformat.yaml.YAMLFactory;
import oshi.json.SystemInfo;
import java.io.File;
import java.io.IOException;
import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@Slf4j
public class SystemPolling {
private ScheduledExecutorService scheduledExecutorService;
private long pollEveryMillis;
private File outputDirectory;
private NameProvider nameProvider;
private ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
private SystemPolling(long pollEveryMillis,File outputDirectory,NameProvider nameProvider) {
this.pollEveryMillis = pollEveryMillis;
this.outputDirectory = outputDirectory;
this.nameProvider = nameProvider;
}
/**
* Run the polling in the background as a thread pool
* running every {@link #pollEveryMillis} milliseconds
*/
public void run() {
scheduledExecutorService = Executors.newScheduledThreadPool(1);
scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
SystemInfo systemInfo = new SystemInfo();
HardwareMetric hardwareMetric = HardwareMetric.fromSystem(systemInfo,nameProvider.nextName());
File hardwareFile = new File(outputDirectory,hardwareMetric.getName() + ".yml");
try {
objectMapper.writeValue(hardwareFile,hardwareMetric);
} catch (IOException e) {
log.error("",e);
}
}
},0,pollEveryMillis, TimeUnit.MILLISECONDS);
}
/**
* Shut down the background polling
*/
public void stopPolling() {
scheduledExecutorService.shutdownNow();
}
/**
* The naming sequence provider.
* This is for allowing generation of naming the output
* according to some semantic sequence (such as a neural net epoch
* or some time stamp)
*/
public interface NameProvider {
String nextName();
}
public static class Builder {
private long pollEveryMillis;
private File outputDirectory;
private NameProvider nameProvider = new NameProvider() {
@Override
public String nextName() {
return UUID.randomUUID().toString();
}
};
/**
* The name provider for this {@link SystemPolling}
* the default value for this is a simple UUID
* @param nameProvider the name provider to use
* @return
*/
public Builder nameProvider(NameProvider nameProvider) {
this.nameProvider = nameProvider;
return this;
}
/**
* The interval in milliseconds in which to poll
* the system for diagnostics
* @param pollEveryMillis the interval in milliseconds
* @return
*/
public Builder pollEveryMillis(long pollEveryMillis) {
this.pollEveryMillis = pollEveryMillis;
return this;
}
/**
* The output directory for the files
* @param outputDirectory the output directory for the logs
* @return
*/
public Builder outputDirectory(File outputDirectory) {
this.outputDirectory = outputDirectory;
return this;
}
public SystemPolling build() {
return new SystemPolling(pollEveryMillis,outputDirectory,nameProvider);
}
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/DataSetLoader.java
+29
View File
@@ -0,0 +1,29 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.loader;
import org.nd4j.common.loader.Loader;
import org.nd4j.common.loader.Source;
import org.nd4j.linalg.dataset.DataSet;
public interface DataSetLoader extends Loader<DataSet> {
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/MultiDataSetLoader.java
+29
View File
@@ -0,0 +1,29 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.loader;
import org.nd4j.common.loader.Loader;
import org.nd4j.common.loader.Source;
import org.nd4j.linalg.dataset.api.MultiDataSet;
public interface MultiDataSetLoader extends Loader<MultiDataSet> {
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/impl/RecordReaderFileBatchLoader.java
+111
View File
@@ -0,0 +1,111 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.loader.impl;
import lombok.Getter;
import lombok.Setter;
import org.datavec.api.records.reader.RecordReader;
import org.datavec.api.records.reader.impl.filebatch.FileBatchRecordReader;
import org.deeplearning4j.core.loader.DataSetLoader;
import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator;
import org.nd4j.common.loader.FileBatch;
import org.nd4j.common.loader.Source;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import java.io.IOException;
public class RecordReaderFileBatchLoader implements DataSetLoader {
private final RecordReader recordReader;
private final int batchSize;
private final int labelIndexFrom;
private final int labelIndexTo;
private final int numPossibleLabels;
private final boolean regression;
@Getter @Setter
private DataSetPreProcessor preProcessor;
/**
* Main constructor for classification. This will convert the input class index (at position labelIndex, with integer
* values 0 to numPossibleLabels-1 inclusive) to the appropriate one-hot output/labels representation.
*
* @param recordReader RecordReader: provides the source of the data
* @param batchSize Batch size (number of examples) for the output DataSet objects
* @param labelIndex Index of the label Writable (usually an IntWritable), as obtained by recordReader.next()
* @param numClasses Number of classes (possible labels) for classification
*/
public RecordReaderFileBatchLoader(RecordReader recordReader, int batchSize, int labelIndex, int numClasses) {
this(recordReader, batchSize, labelIndex, labelIndex, numClasses, false, null);
}
/**
* Main constructor for multi-label regression (i.e., regression with multiple outputs). Can also be used for single
* output regression with labelIndexFrom == labelIndexTo
*
* @param recordReader RecordReader to get data from
* @param labelIndexFrom Index of the first regression target
* @param labelIndexTo Index of the last regression target, inclusive
* @param batchSize Minibatch size
* @param regression Require regression = true. Mainly included to avoid clashing with other constructors previously defined :/
*/
public RecordReaderFileBatchLoader(RecordReader recordReader, int batchSize, int labelIndexFrom, int labelIndexTo,
boolean regression) {
this(recordReader, batchSize, labelIndexFrom, labelIndexTo, -1, regression, null);
}
/**
* Main constructor
*
* @param recordReader the recordreader to use
* @param batchSize Minibatch size - number of examples returned for each call of .next()
* @param labelIndexFrom the index of the label (for classification), or the first index of the labels for multi-output regression
* @param labelIndexTo only used if regression == true. The last index <i>inclusive</i> of the multi-output regression
* @param numPossibleLabels the number of possible labels for classification. Not used if regression == true
* @param regression if true: regression. If false: classification (assume labelIndexFrom is the class it belongs to)
* @param preProcessor Optional DataSetPreProcessor. May be null.
*/
public RecordReaderFileBatchLoader(RecordReader recordReader, int batchSize, int labelIndexFrom, int labelIndexTo,
int numPossibleLabels, boolean regression, DataSetPreProcessor preProcessor) {
this.recordReader = recordReader;
this.batchSize = batchSize;
this.labelIndexFrom = labelIndexFrom;
this.labelIndexTo = labelIndexTo;
this.numPossibleLabels = numPossibleLabels;
this.regression = regression;
this.preProcessor = preProcessor;
}
@Override
public DataSet load(Source source) throws IOException {
FileBatch fb = FileBatch.readFromZip(source.getInputStream());
//Wrap file batch in RecordReader
//Create RecordReaderDataSetIterator
//Return dataset
RecordReader rr = new FileBatchRecordReader(recordReader, fb);
RecordReaderDataSetIterator iter = new RecordReaderDataSetIterator(rr, null, batchSize, labelIndexFrom, labelIndexTo, numPossibleLabels, -1, regression);
if (preProcessor != null) {
iter.setPreProcessor(preProcessor);
}
DataSet ds = iter.next();
return ds;
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/impl/SerializedDataSetLoader.java
+39
View File
@@ -0,0 +1,39 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.loader.impl;
import org.deeplearning4j.core.loader.DataSetLoader;
import org.nd4j.common.loader.Source;
import org.nd4j.linalg.dataset.DataSet;
import java.io.IOException;
import java.io.InputStream;
public class SerializedDataSetLoader implements DataSetLoader {
@Override
public DataSet load(Source source) throws IOException {
DataSet ds = new DataSet();
try(InputStream is = source.getInputStream()){
ds.load(is);
}
return ds;
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/impl/SerializedMultiDataSetLoader.java
+39
View File
@@ -0,0 +1,39 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.loader.impl;
import org.deeplearning4j.core.loader.MultiDataSetLoader;
import org.nd4j.common.loader.Source;
import org.nd4j.linalg.dataset.api.MultiDataSet;
import java.io.IOException;
import java.io.InputStream;
public class SerializedMultiDataSetLoader implements MultiDataSetLoader {
@Override
public MultiDataSet load(Source source) throws IOException {
org.nd4j.linalg.dataset.MultiDataSet ds = new org.nd4j.linalg.dataset.MultiDataSet();
try(InputStream is = source.getInputStream()){
ds.load(is);
}
return ds;
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/parallelism/AsyncIterator.java
+141
View File
@@ -0,0 +1,141 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.parallelism;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import java.util.Iterator;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
@Slf4j
public class AsyncIterator<T extends Object> implements Iterator<T> {
@Getter
protected BlockingQueue<T> buffer;
protected ReaderThread<T> thread;
protected Iterator<T> iterator;
@Getter
protected T terminator = (T) new Object();
protected T nextElement;
protected AtomicBoolean shouldWork = new AtomicBoolean(true);
public AsyncIterator(@NonNull Iterator<T> iterator, int bufferSize) {
this.buffer = new LinkedBlockingQueue<>(bufferSize);
this.iterator = iterator;
thread = new ReaderThread<>(iterator, this.buffer, terminator);
thread.start();
}
public AsyncIterator(@NonNull Iterator<T> iterator) {
this(iterator, 1024);
}
@Override
public boolean hasNext() {
try {
if (nextElement != null && nextElement != terminator) {
return true;
}
// if on previous run we've got terminator - just return false
if (nextElement == terminator)
return false;
nextElement = buffer.take();
// same on this run
return (nextElement != terminator);
} catch (Exception e) {
log.error("Premature end of loop!");
return false;
}
}
@Override
public T next() {
T temp = nextElement;
nextElement = temp == terminator ? terminator : null;
return temp;
}
@Override
public void remove() {
// no-op
}
public void shutdown() {
if (shouldWork.get()) {
shouldWork.set(false);
thread.interrupt();
try {
// Shutdown() should be a synchronous operation since the iterator is reset after shutdown() is
// called in AsyncLabelAwareIterator.reset().
thread.join();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
nextElement = terminator;
buffer.clear();
}
}
private class ReaderThread<T> extends Thread implements Runnable {
private BlockingQueue<T> buffer;
private Iterator<T> iterator;
private T terminator;
public ReaderThread(Iterator<T> iterator, BlockingQueue<T> buffer, T terminator) {
this.buffer = buffer;
this.iterator = iterator;
this.terminator = terminator;
setDaemon(true);
setName("AsyncIterator Reader thread");
}
@Override
public void run() {
//log.info("AsyncReader [{}] started", Thread.currentThread().getId());
try {
while (iterator.hasNext() && shouldWork.get()) {
T smth = iterator.next();
if (smth != null)
buffer.put(smth);
}
buffer.put(terminator);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
// do nothing
shouldWork.set(false);
} catch (Exception e) {
// TODO: pass that forward
throw new RuntimeException(e);
} finally {
//log.info("AsyncReader [{}] stopped", Thread.currentThread().getId());
}
}
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/Persistable.java
+100
View File
@@ -0,0 +1,100 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.nio.ByteBuffer;
public interface Persistable extends Serializable {
/**
* Get the session id
* @return
*/
String getSessionID();
/**
* Get the type id
* @return
*/
String getTypeID();
/**
* Get the worker id
* @return
*/
String getWorkerID();
/**
* Get when this was created.
* @return
*/
long getTimeStamp();
//SerDe methods:
/**
* Length of the encoding, in bytes, when using {@link #encode()}
* Length may be different using {@link #encode(OutputStream)}, due to things like stream headers
* @return
*/
int encodingLengthBytes();
byte[] encode();
/**
* Encode this persistable in to a {@link ByteBuffer}
* @param buffer
*/
void encode(ByteBuffer buffer);
/**
* Encode this persistable in to an output stream
* @param outputStream
* @throws IOException
*/
void encode(OutputStream outputStream) throws IOException;
/**
* Decode the content of the given
* byte array in to this persistable
* @param decode
*/
void decode(byte[] decode);
/**
* Decode from the given {@link ByteBuffer}
* @param buffer
*/
void decode(ByteBuffer buffer);
/**
* Decode from the given input stream
* @param inputStream
* @throws IOException
*/
void decode(InputStream inputStream) throws IOException;
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorage.java
+220
View File
@@ -0,0 +1,220 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
import java.io.IOException;
import java.util.List;
public interface StatsStorage extends StatsStorageRouter {
/**
* Close any open resources (files, etc)
*/
void close() throws IOException;
/**
* @return Whether the StatsStorage implementation has been closed or not
*/
boolean isClosed();
/**
* Get a list of all sessions stored by this storage backend
*/
List<String> listSessionIDs();
/**
* Check if the specified session ID exists or not
*
* @param sessionID Session ID to check
* @return true if session exists, false otherwise
*/
boolean sessionExists(String sessionID);
/**
* Get the static info for the given session and worker IDs, or null if no such static info has been reported
*
* @param sessionID Session ID
* @param workerID worker ID
* @return Static info, or null if none has been reported
*/
Persistable getStaticInfo(String sessionID, String typeID, String workerID);
/**
* Get all static informations for the given session and type ID
*
* @param sessionID Session ID to get static info for
* @param typeID Type ID to get static info for
* @return All static info instances matching both the session and type IDs
*/
List<Persistable> getAllStaticInfos(String sessionID, String typeID);
/**
* Get the list of type IDs for the given session ID
*
* @param sessionID Session ID to query
* @return List of type IDs
*/
List<String> listTypeIDsForSession(String sessionID);
/**
* For a given session ID, list all of the known worker IDs
*
* @param sessionID Session ID
* @return List of worker IDs, or possibly null if session ID is unknown
*/
List<String> listWorkerIDsForSession(String sessionID);
/**
* For a given session ID and type ID, list all of the known worker IDs
*
* @param sessionID Session ID
* @param typeID Type ID
* @return List of worker IDs, or possibly null if session ID is unknown
*/
List<String> listWorkerIDsForSessionAndType(String sessionID, String typeID);
/**
* Return the number of update records for the given session ID (all workers)
*
* @param sessionID Session ID
* @return number of update records
*/
int getNumUpdateRecordsFor(String sessionID);
/**
* Return the number of update records for the given session ID and worker ID
*
* @param sessionID Session ID
* @param workerID Worker ID
* @return number of update records
*/
int getNumUpdateRecordsFor(String sessionID, String typeID, String workerID);
/**
* Get the latest update record (i.e., update record with the largest timestamp value) for the specified
* session and worker IDs
*
* @param sessionID session ID
* @param workerID worker ID
* @return UpdateRecord containing the session/worker IDs, timestamp and content for the most recent update
*/
Persistable getLatestUpdate(String sessionID, String typeID, String workerID);
/**
* Get the specified update (or null, if none exists for the given session/worker ids and timestamp)
*
* @param sessionID Session ID
* @param workerID Worker ID
* @param timestamp Timestamp
* @return Update
*/
Persistable getUpdate(String sessionID, String typeId, String workerID, long timestamp);
/**
* Get the latest update for all workers, for the given session ID
*
* @param sessionID Session ID
* @return List of updates for the given Session ID
*/
List<Persistable> getLatestUpdateAllWorkers(String sessionID, String typeID);
/**
* Get all updates for the given session and worker ID, that occur after (not including) the given timestamp.
* Results should be sorted by time.
*
* @param sessionID Session ID
* @param workerID Worker Id
* @param timestamp Timestamp
* @return List of records occurring after the given timestamp
*/
List<Persistable> getAllUpdatesAfter(String sessionID, String typeID, String workerID, long timestamp);
/**
* Get all updates for the given session ID (all worker IDs), that occur after (not including) the given timestamp.
* Results should be sorted by time.
*
* @param sessionID Session ID
* @param timestamp Timestamp
* @return List of records occurring after the given timestamp
*/
List<Persistable> getAllUpdatesAfter(String sessionID, String typeID, long timestamp);
/**
* List the times of all updates for the specified sessionID, typeID and workerID
*
* @param sessionID Session ID to get update times for
* @param typeID Type ID to get update times for
* @param workerID Worker ID to get update times for
* @return Times of all updates
*/
long[] getAllUpdateTimes(String sessionID, String typeID, String workerID);
/**
* Get updates for the specified times only
*
* @param sessionID Session ID to get update times for
* @param typeID Type ID to get update times for
* @param workerID Worker ID to get update times for
* @param timestamps Timestamps to get the updates for. Note that if one of the specified times does not exist,
* it will be ommitted from the returned results list.
* @return List of updates at the specified times
*/
List<Persistable> getUpdates(String sessionID, String typeID, String workerID, long[] timestamps);
/**
* Get the session metadata, if any has been registered via {@link #putStorageMetaData(StorageMetaData)}
*
* @param sessionID Session ID to get metadat
* @return Session metadata, or null if none is available
*/
StorageMetaData getStorageMetaData(String sessionID, String typeID);
// ----- Listeners -----
/**
* Add a new StatsStorageListener. The given listener will called whenever a state change occurs for the stats
* storage instance
*
* @param listener Listener to add
*/
void registerStatsStorageListener(StatsStorageListener listener);
/**
* Remove the specified listener, if it is present.
*
* @param listener Listener to remove
*/
void deregisterStatsStorageListener(StatsStorageListener listener);
/**
* Remove all listeners from the StatsStorage instance
*/
void removeAllListeners();
/**
* Get a list (shallow copy) of all listeners currently present
*
* @return List of listeners
*/
List<StatsStorageListener> getListeners();
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageEvent.java
+35
View File
@@ -0,0 +1,35 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
import lombok.AllArgsConstructor;
import lombok.Data;
@AllArgsConstructor
@Data
public class StatsStorageEvent {
private final StatsStorage statsStorage;
private final StatsStorageListener.EventType eventType;
private final String sessionID;
private final String typeID;
private final String workerID;
private final long timestamp;
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageListener.java
+37
View File
@@ -0,0 +1,37 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
public interface StatsStorageListener {
enum EventType {
NewSessionID, NewTypeID, NewWorkerID, PostMetaData, PostStaticInfo, PostUpdate
}
/**
* Notify will be called whenever an event (new information posted, etc) occurs.
* Processing these events should ideally be done asynchronously.
*
* @param event Event that occurred
*/
void notify(StatsStorageEvent event);
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageRouter.java
+68
View File
@@ -0,0 +1,68 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
import java.util.Collection;
public interface StatsStorageRouter {
/**
* Method to store some additional metadata for each session. Idea: record the classes used to
* serialize and deserialize the static info and updates (as a class name).
* This is mainly used for debugging and validation.
*
* @param storageMetaData Storage metadata to store
*/
void putStorageMetaData(StorageMetaData storageMetaData); //TODO error handling
void putStorageMetaData(Collection<? extends StorageMetaData> storageMetaData);
/**
* Static info: reported once per session, upon initialization
*
* @param staticInfo Static info to store
*/
void putStaticInfo(Persistable staticInfo); //TODO error handling
/**
* Static info: reported once per session, upon initialization
*
* @param staticInfo Static info to store
*/
void putStaticInfo(Collection<? extends Persistable> staticInfo);
/**
* Updates: stored multiple times per session (periodically, for example)
*
* @param update Update info to store
*/
void putUpdate(Persistable update); //TODO error handling
/**
* Updates: stored multiple times per session (periodically, for example)
*
* @param updates Update info to store
*/
void putUpdate(Collection<? extends Persistable> updates);
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageRouterProvider.java
+29
View File
@@ -0,0 +1,29 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
import java.io.Serializable;
public interface StatsStorageRouterProvider extends Serializable {
StatsStorageRouter getRouter();
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StorageMetaData.java
+62
View File
@@ -0,0 +1,62 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
import java.io.Serializable;
public interface StorageMetaData extends Persistable {
/**
* Timestamp for the metadata
*/
long getTimeStamp();
/**
* Session ID for the metadata
*/
String getSessionID();
/**
* Type ID for the metadata
*/
String getTypeID();
/**
* Worker ID for the metadata
*/
String getWorkerID();
/**
* Full class name for the initialization information that will be posted. Is expected to implement {@link Persistable}.
*/
String getInitTypeClass();
/**
* Full class name for the update information that will be posted. Is expected to implement {@link Persistable}.
*/
String getUpdateTypeClass();
/**
* Get extra metadata, if any
*/
Serializable getExtraMetaData();
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StorageType.java
+25
View File
@@ -0,0 +1,25 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
public enum StorageType {
MetaData, StaticInfo, Update
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/impl/CollectionStatsStorageRouter.java
+67
View File
@@ -0,0 +1,67 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage.impl;
import lombok.AllArgsConstructor;
import org.deeplearning4j.core.storage.Persistable;
import org.deeplearning4j.core.storage.StatsStorageRouter;
import org.deeplearning4j.core.storage.StorageMetaData;
import java.util.Collection;
@AllArgsConstructor
public class CollectionStatsStorageRouter implements StatsStorageRouter {
private Collection<StorageMetaData> metaDatas;
private Collection<Persistable> staticInfos;
private Collection<Persistable> updates;
@Override
public void putStorageMetaData(StorageMetaData storageMetaData) {
this.metaDatas.add(storageMetaData);
}
@Override
public void putStorageMetaData(Collection<? extends StorageMetaData> storageMetaData) {
this.metaDatas.addAll(storageMetaData);
}
@Override
public void putStaticInfo(Persistable staticInfo) {
this.staticInfos.add(staticInfo);
}
@Override
public void putStaticInfo(Collection<? extends Persistable> staticInfo) {
this.staticInfos.addAll(staticInfo);
}
@Override
public void putUpdate(Persistable update) {
this.updates.add(update);
}
@Override
public void putUpdate(Collection<? extends Persistable> updates) {
this.updates.addAll(updates);
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/impl/RemoteUIStatsStorageRouter.java
+390
View File
@@ -0,0 +1,390 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage.impl;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.binary.Base64;
import org.deeplearning4j.core.storage.Persistable;
import org.deeplearning4j.core.storage.StatsStorageRouter;
import org.deeplearning4j.core.storage.StorageMetaData;
import org.deeplearning4j.core.storage.StorageType;
import org.nd4j.shade.jackson.databind.ObjectMapper;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.*;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@Slf4j
public class RemoteUIStatsStorageRouter implements StatsStorageRouter, Serializable, Closeable {
private static final String ROUTE_IS_DOWN = "Info posted to RemoteUIStatsStorageRouter but router is shut down.";
private static final String MAX_WARNINGS_REACHED = "RemoteUIStatsStorageRouter: Reached max shutdown warnings. No further warnings will be produced.";
/**
* Default path for posting data to the UI - i.e., http://localhost:9000/remoteReceive or similar
*/
public static final String DEFAULT_PATH = "remoteReceive";
/**
* Default maximum number of (consecutive) retries on failure
*/
public static final int DEFAULT_MAX_RETRIES = 10;
/**
* Base delay for retries
*/
public static final long DEFAULT_BASE_RETR_DELAY_MS = 1000;
/**
* Default backoff multiplicative factor for retrying
*/
public static final double DEFAULT_RETRY_BACKOFF_FACTOR = 2.0;
private static final long MAX_SHUTDOWN_WARN_COUNT = 5;
private final String USER_AGENT = "Mozilla/5.0";
private final URL url;
private final int maxRetryCount;
private final long retryDelayMS;
private final double retryBackoffFactor;
private transient LinkedBlockingDeque<ToPost> queue = new LinkedBlockingDeque<>();
private transient Thread postThread;
private AtomicBoolean shutdown = new AtomicBoolean(false);
private AtomicLong shutdownWarnCount = new AtomicLong(0);
private static final ObjectMapper objectMapper = new ObjectMapper();
/**
* Create remote UI with defaults for all values except address
*
* @param address Address of the remote UI: for example, "http://localhost:9000"
*/
public RemoteUIStatsStorageRouter(String address) {
this(address, DEFAULT_MAX_RETRIES, DEFAULT_BASE_RETR_DELAY_MS, DEFAULT_RETRY_BACKOFF_FACTOR);
}
/**
* @param address Address of the remote UI: for example, "http://localhost:9000"
* @param maxRetryCount Maximum number of retries before failing. Set to -1 to always retry
* @param retryDelayMS Base delay before retrying, in milliseconds
* @param retryBackoffFactor Backoff factor for retrying: 2.0 for example gives delays of 1000, 2000, 4000, 8000,
* etc milliseconds, with a base retry delay of 1000
*/
public RemoteUIStatsStorageRouter(String address, int maxRetryCount, long retryDelayMS, double retryBackoffFactor) {
this(address, DEFAULT_PATH, maxRetryCount, retryDelayMS, retryBackoffFactor);
}
/**
* @param address Address of the remote UI: for example, "http://localhost:9000"
* @param path Path/endpoint to post to: for example "remoteReceive" -> added to path to become like
* "http://localhost:9000/remoteReceive"
* @param maxRetryCount Maximum number of retries before failing. Set to -1 to always retry
* @param retryDelayMS Base delay before retrying, in milliseconds
* @param retryBackoffFactor Backoff factor for retrying: 2.0 for example gives delays of 1000, 2000, 4000, 8000,
* etc milliseconds, with a base retry delay of 1000
*/
public RemoteUIStatsStorageRouter(String address, String path, int maxRetryCount, long retryDelayMS,
double retryBackoffFactor) {
this.maxRetryCount = maxRetryCount;
this.retryDelayMS = retryDelayMS;
this.retryBackoffFactor = retryBackoffFactor;
String url = address;
if (path != null) {
if (url.endsWith("/")) {
url = url + path;
} else {
url = url + "/" + path;
}
}
try {
this.url = new URL(url);
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}
}
@Override
public void close(){
shutdown();
}
public void shutdown(){
this.shutdown.set(true);
}
private synchronized void checkThread(){
if(postThread == null){
postThread = new Thread(new PostRunnable());
postThread.setDaemon(true);
postThread.start();
}
if(queue == null){
//May be null if router has been deserialized
queue = new LinkedBlockingDeque<>();
}
}
@Override
public void putStorageMetaData(StorageMetaData storageMetaData) {
putStorageMetaData(Collections.singleton(storageMetaData));
}
@Override
public void putStorageMetaData(Collection<? extends StorageMetaData> storageMetaData) {
checkThread();
if (shutdown.get()) {
long count = shutdownWarnCount.getAndIncrement();
if (count <= MAX_SHUTDOWN_WARN_COUNT) {
log.warn(ROUTE_IS_DOWN);
}
if (count == MAX_SHUTDOWN_WARN_COUNT) {
log.warn(MAX_WARNINGS_REACHED);
}
} else {
for (StorageMetaData m : storageMetaData) {
queue.add(new ToPost(m, null, null));
}
}
}
@Override
public void putStaticInfo(Persistable staticInfo) {
putStaticInfo(Collections.singletonList(staticInfo));
}
@Override
public void putStaticInfo(Collection<? extends Persistable> staticInfo) {
checkThread();
if (shutdown.get()) {
long count = shutdownWarnCount.getAndIncrement();
if (count <= MAX_SHUTDOWN_WARN_COUNT) {
log.warn(ROUTE_IS_DOWN);
}
if (count == MAX_SHUTDOWN_WARN_COUNT) {
log.warn(MAX_WARNINGS_REACHED);
}
} else {
for (Persistable p : staticInfo) {
queue.add(new ToPost(null, p, null));
}
}
}
@Override
public void putUpdate(Persistable update) {
putUpdate(Collections.singleton(update));
}
@Override
public void putUpdate(Collection<? extends Persistable> updates) {
checkThread();
if (shutdown.get()) {
long count = shutdownWarnCount.getAndIncrement();
if (count <= MAX_SHUTDOWN_WARN_COUNT) {
log.warn(ROUTE_IS_DOWN);
}
if (count == MAX_SHUTDOWN_WARN_COUNT) {
log.warn(MAX_WARNINGS_REACHED);
}
} else {
for (Persistable p : updates) {
queue.add(new ToPost(null, null, p));
}
}
}
@AllArgsConstructor
@Data
private static class ToPost {
private final StorageMetaData meta;
private final Persistable staticInfo;
private final Persistable update;
}
//Runnable class for doing async posting
private class PostRunnable implements Runnable {
private int failureCount = 0;
private long nextDelayMs = retryDelayMS;
@Override
public void run() {
try {
runHelper();
} catch (Exception e) {
log.error("Exception encountered in remote UI posting thread. Shutting down.", e);
shutdown.set(true);
}
}
private void runHelper() {
while (!shutdown.get()) {
List<ToPost> list = new ArrayList<>();
ToPost t;
try {
t = queue.take(); //Blocking operation
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
continue;
}
list.add(t);
queue.drainTo(list); //Non-blocking
int successCount = 0;
for (ToPost toPost : list) {
boolean success;
try {
success = tryPost(toPost);
} catch (IOException e) {
failureCount++;
log.warn("Error posting to remote UI at {}, consecutive failure count = {}. Waiting {} ms before retrying",
url, failureCount, nextDelayMs, e);
success = false;
}
if (!success) {
for (int i = list.size() - 1; i > successCount; i--) {
queue.addFirst(list.get(i)); //Add remaining back to be processed in original order
}
waitForRetry();
break;
} else {
successCount++;
failureCount = 0;
nextDelayMs = retryDelayMS;
}
}
}
}
private void waitForRetry() {
if (maxRetryCount >= 0 && failureCount > maxRetryCount) {
throw new RuntimeException("RemoteUIStatsStorageRouter: hit maximum consecutive failures("
+ maxRetryCount + "). Shutting down remote router thread");
} else {
try {
Thread.sleep(nextDelayMs);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
nextDelayMs *= retryBackoffFactor;
}
}
}
private HttpURLConnection getConnection() throws IOException {
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("POST");
connection.setRequestProperty("User-Agent", USER_AGENT);
connection.setRequestProperty("Content-Type", "application/json");
connection.setDoOutput(true);
return connection;
}
private boolean tryPost(ToPost toPost) throws IOException {
HttpURLConnection connection = getConnection();
String className;
byte[] asBytes;
StorageType type;
if (toPost.getMeta() != null) {
StorageMetaData smd = toPost.getMeta();
className = smd.getClass().getName();
asBytes = smd.encode();
type = StorageType.MetaData;
} else if (toPost.getStaticInfo() != null) {
Persistable p = toPost.getStaticInfo();
className = p.getClass().getName();
asBytes = p.encode();
type = StorageType.StaticInfo;
} else {
Persistable p = toPost.getUpdate();
className = p.getClass().getName();
asBytes = p.encode();
type = StorageType.Update;
}
String base64 = Base64.encodeBase64String(asBytes);
Map<String, String> jsonObj = new LinkedHashMap<>();
jsonObj.put("type", type.name());
jsonObj.put("class", className);
jsonObj.put("data", base64);
String str;
try {
str = objectMapper.writeValueAsString(jsonObj);
} catch (Exception e) {
throw new RuntimeException(e); //Should never get an exception from simple Map<String,String>
}
DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
dos.writeBytes(str);
dos.flush();
dos.close();
try {
int responseCode = connection.getResponseCode();
if (responseCode != 200) {
BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
String inputLine;
StringBuilder response = new StringBuilder();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
log.warn("Error posting to remote UI - received response code {}\tContent: {}", response,
response.toString());
return false;
}
} catch (IOException e) {
String msg = e.getMessage();
if (msg.contains("403 for URL")) {
log.warn("Error posting to remote UI at {} (Response code: 403)."
+ " Remote listener support is not enabled? use UIServer.getInstance().enableRemoteListener()",
url, e);
} else {
log.warn("Error posting to remote UI at {}", url, e);
}
return false;
}
return true;
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/listener/RoutingIterationListener.java
+45
View File
@@ -0,0 +1,45 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage.listener;
import org.deeplearning4j.core.storage.StatsStorageRouter;
import org.deeplearning4j.core.storage.StatsStorageRouterProvider;
import org.deeplearning4j.optimize.api.TrainingListener;
import java.io.Serializable;
public interface RoutingIterationListener extends TrainingListener, Cloneable, Serializable {
void setStorageRouter(StatsStorageRouter router);
StatsStorageRouter getStorageRouter();
void setWorkerID(String workerID);
String getWorkerID();
void setSessionID(String sessionID);
String getSessionID();
RoutingIterationListener clone();
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/ui/UiConnectionInfo.java
+150
View File
@@ -0,0 +1,150 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.ui;
import lombok.Data;
import lombok.NonNull;
@Data
public class UiConnectionInfo {
private String sessionId;
private String login;
private String password;
private String address = "localhost";
private int port = 8080;
private String path = "";
private boolean useHttps;
public UiConnectionInfo() {
this.sessionId = java.util.UUID.randomUUID().toString();
}
public void setSessionId(@NonNull String sessionId) {
this.sessionId = sessionId;
}
/**
* This method returns scheme, address and port for this UiConnectionInfo
*
* i.e: https://localhost:8080
*
* @return
*/
public String getFirstPart() {
StringBuilder builder = new StringBuilder();
builder.append(useHttps ? "https" : "http").append("://").append(address).append(":").append(port).append("");
return builder.toString();
}
public String getSecondPart() {
return getSecondPart("");
}
public String getSecondPart(String nPath) {
StringBuilder builder = new StringBuilder();
if (path != null && !path.isEmpty()) {
builder.append(path.startsWith("/") ? path : ("/" + path)).append("/");
}
if (nPath != null) {
nPath = nPath.replaceFirst("^/", "");
builder.append(nPath.startsWith("/") ? nPath : ("/" + nPath)).append("/");
}
return builder.toString().replaceAll("\\/{2,}", "/");
}
public String getFullAddress(String nPath) {
if (nPath == null || nPath.isEmpty()) {
return getFullAddress();
} else {
return getFirstPart() + getSecondPart(nPath) + "?sid=" + this.getSessionId();
}
}
public String getFullAddress() {
return getFirstPart() + getSecondPart();
}
public static class Builder {
private UiConnectionInfo info = new UiConnectionInfo();
/**
* This method allows you to specify sessionId for this UiConnectionInfo instance
*
* PLEASE NOTE: This is not recommended. Advised behaviour - keep it random, as is.
*
* @param sessionId
* @return
*/
public Builder setSessionId(@NonNull String sessionId) {
info.setSessionId(sessionId);
return this;
}
public Builder setLogin(@NonNull String login) {
info.setLogin(login);
return this;
}
public Builder setPassword(String password) {
info.setPassword(password);
return this;
}
public Builder setAddress(@NonNull String address) {
info.setAddress(address);
return this;
}
public Builder setPort(int port) {
if (port <= 0)
throw new IllegalStateException("UiServer port can't be <= 0");
info.setPort(port);
return this;
}
public Builder enableHttps(boolean reallyEnable) {
info.setUseHttps(reallyEnable);
return this;
}
/**
* If you're using UiServer as servlet, located not at root folder of webserver (i.e. http://yourdomain.com/somepath/webui/), you can set path here.
* For provided example path will be "/somepath/webui/"
*
* @param path
* @return
*/
public Builder setPath(String path) {
info.setPath(path);
return this;
}
public UiConnectionInfo build() {
return info;
}
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/ModelGuesser.java
+222
View File
@@ -0,0 +1,222 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.util;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.deeplearning4j.common.util.ND4JFileUtils;
import org.deeplearning4j.config.DL4JSystemProperties;
import org.deeplearning4j.nn.api.Model;
import org.deeplearning4j.nn.conf.ComputationGraphConfiguration;
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.modelimport.keras.KerasModelImport;
import org.deeplearning4j.util.ModelSerializer;
import org.nd4j.linalg.dataset.api.preprocessor.Normalizer;
import java.io.*;
import java.util.UUID;
/**
* Guess a model from the given path
* @author Adam Gibson
*/
@Slf4j
public class ModelGuesser {
/**
* A facade for {@link ModelSerializer#restoreNormalizerFromInputStream(InputStream)}
* @param is the input stream to load form
* @return the loaded normalizer
* @throws IOException
*/
public static Normalizer<?> loadNormalizer(InputStream is) throws IOException {
return ModelSerializer.restoreNormalizerFromInputStream(is);
}
/**
* A facade for {@link ModelSerializer#restoreNormalizerFromFile(File)}
* @param path the path to the file
* @return the loaded normalizer
*/
public static Normalizer<?> loadNormalizer(String path) {
try {
return ModelSerializer.restoreNormalizerFromFile(new File(path));
} catch (IOException e){
throw new RuntimeException(e);
}
}
/**
* Load the model from the given file path
* @param path the path of the file to "guess"
*
* @return the loaded model
* @throws Exception
*/
public static Object loadConfigGuess(String path) throws Exception {
String input = FileUtils.readFileToString(new File(path));
//note here that we load json BEFORE YAML. YAML
//turns out to load just fine *accidentally*
try {
return MultiLayerConfiguration.fromJson(input);
} catch (Exception e) {
log.warn("Tried multi layer config from json", e);
try {
return KerasModelImport.importKerasModelConfiguration(path);
} catch (Exception e1) {
log.warn("Tried keras model config", e);
try {
return KerasModelImport.importKerasSequentialConfiguration(path);
} catch (Exception e2) {
log.warn("Tried keras sequence config", e);
try {
return ComputationGraphConfiguration.fromJson(input);
} catch (Exception e3) {
log.warn("Tried computation graph from json");
try {
return MultiLayerConfiguration.fromYaml(input);
} catch (Exception e4) {
log.warn("Tried multi layer configuration from yaml");
try {
return ComputationGraphConfiguration.fromYaml(input);
} catch (Exception e5) {
throw new ModelGuesserException("Unable to load configuration from path " + path
+ " (invalid config file or not a known config type)");
}
}
}
}
}
}
}
/**
* Load the model from the given input stream
* @param stream the path of the file to "guess"
*
* @return the loaded model
* @throws Exception
*/
public static Object loadConfigGuess(InputStream stream) throws Exception {
String p = System.getProperty(DL4JSystemProperties.DL4J_TEMP_DIR_PROPERTY);
File tmp = ND4JFileUtils.createTempFile("model-" + UUID.randomUUID().toString(), "bin");
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(tmp));
IOUtils.copy(stream, bufferedOutputStream);
bufferedOutputStream.flush();
bufferedOutputStream.close();
tmp.deleteOnExit();
try {
return loadConfigGuess(tmp.getAbsolutePath());
} finally {
tmp.delete();
}
}
/**
* Load the model from the given file path
* @param path the path of the file to "guess"
*
* @return the loaded model
* @throws Exception
*/
public static Model loadModelGuess(String path) throws Exception {
try {
return ModelSerializer.restoreMultiLayerNetwork(new File(path), true);
} catch (Exception e) {
log.warn("Tried multi layer network");
try {
return ModelSerializer.restoreComputationGraph(new File(path), true);
} catch (Exception e1) {
log.warn("Tried computation graph");
try {
return ModelSerializer.restoreMultiLayerNetwork(new File(path), false);
} catch (Exception e4) {
try {
return ModelSerializer.restoreComputationGraph(new File(path), false);
} catch (Exception e5) {
try {
return KerasModelImport.importKerasModelAndWeights(path);
} catch (Exception e2) {
log.warn("Tried multi layer network keras");
try {
return KerasModelImport.importKerasSequentialModelAndWeights(path);
} catch (Exception e3) {
throw new ModelGuesserException("Unable to load model from path " + path
+ " (invalid model file or not a known model type)");
}
}
}
}
}
}
}
/**
* Load the model from the given input stream. The content of the stream is written to a temporary location,
* see {@link DL4JSystemProperties#DL4J_TEMP_DIR_PROPERTY}
*
* @param stream the path of the file to "guess"
*
* @return the loaded model
* @throws Exception
*/
public static Model loadModelGuess(InputStream stream) throws Exception {
return loadModelGuess(stream, null);
}
/**
* As per {@link #loadModelGuess(InputStream)} but (optionally) allows copying to the specified temporary directory
* @param stream Stream of the model file
* @param tempDirectory Temporary/working directory. May be null.
*/
public static Model loadModelGuess(InputStream stream, File tempDirectory) throws Exception {
//Currently (Nov 2017): KerasModelImport doesn't support loading from input streams
//Simplest solution here: write to a temporary file
File f;
if(tempDirectory == null){
f = ND4JFileUtils.createTempFile("loadModelGuess",".bin");
} else {
f = File.createTempFile("loadModelGuess", ".bin", tempDirectory);
}
f.deleteOnExit();
try (OutputStream os = new BufferedOutputStream(new FileOutputStream(f))) {
IOUtils.copy(stream, os);
os.flush();
return loadModelGuess(f.getAbsolutePath());
} catch (ModelGuesserException e){
throw new ModelGuesserException("Unable to load model from input stream (invalid model file not a known model type)");
} finally {
f.delete();
}
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/ModelGuesserException.java
+36
View File
@@ -0,0 +1,36 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.util;
public class ModelGuesserException extends Exception {
public ModelGuesserException(String message) {
super(message);
}
public ModelGuesserException(String message, Throwable cause) {
super(message, cause);
}
public ModelGuesserException(Throwable cause) {
super(cause);
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/MovingWindowMatrix.java
+124
View File
@@ -0,0 +1,124 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.util;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.factory.Nd4j;
import java.util.ArrayList;
import java.util.List;
/**
*
* Moving window on a matrix (usually used for images)
*
* Given a: This is a list of flattened arrays:
* 1 1 1 1 1 1 2 2
* 2 2 2 2 ----> 1 1 2 2
* 3 3 3 3 3 3 4 4
* 4 4 4 4 3 3 4 4
*
* @author Adam Gibson
*/
public class MovingWindowMatrix {
private int windowRowSize = 28;
private int windowColumnSize = 28;
private INDArray toSlice;
private boolean addRotate = false;
/**
*
* @param toSlice matrix to slice
* @param windowRowSize the number of rows in each window
* @param windowColumnSize the number of columns in each window
* @param addRotate whether to add the possible rotations of each moving window
*/
public MovingWindowMatrix(INDArray toSlice, int windowRowSize, int windowColumnSize, boolean addRotate) {
this.toSlice = toSlice;
this.windowRowSize = windowRowSize;
this.windowColumnSize = windowColumnSize;
this.addRotate = addRotate;
}
/**
* Same as calling new MovingWindowMatrix(toSlice,windowRowSize,windowColumnSize,false)
* @param toSlice
* @param windowRowSize
* @param windowColumnSize
*/
public MovingWindowMatrix(INDArray toSlice, int windowRowSize, int windowColumnSize) {
this(toSlice, windowRowSize, windowColumnSize, false);
}
/**
* Returns a list of non flattened moving window matrices
* @return the list of matrices
*/
public List<INDArray> windows() {
return windows(false);
}
/**
* Moving window, capture a row x column moving window of
* a given matrix
* @param flattened whether the arrays should be flattened or not
* @return the list of moving windows
*/
public List<INDArray> windows(boolean flattened) {
List<INDArray> ret = new ArrayList<>();
int window = 0;
for (int i = 0; i < toSlice.length(); i++) {
if (window >= toSlice.length())
break;
double[] w = new double[this.windowRowSize * this.windowColumnSize];
for (int count = 0; count < this.windowRowSize * this.windowColumnSize; count++) {
w[count] = toSlice.getDouble(count + window);
}
INDArray add = Nd4j.create(w);
if (flattened)
add = add.ravel();
else
add = add.reshape(windowRowSize, windowColumnSize);
if (addRotate) {
INDArray currRotation = add.dup();
//3 different orientations besides the original
for (int rotation = 0; rotation < 3; rotation++) {
Nd4j.rot90(currRotation);
ret.add(currRotation.dup());
}
}
window += this.windowRowSize * this.windowColumnSize;
ret.add(add);
}
return ret;
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/ThreadUtils.java
+51
View File
@@ -0,0 +1,51 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.util;
import java.util.concurrent.locks.LockSupport;
public class ThreadUtils {
public static void uncheckedSleep(long millis) {
LockSupport.parkNanos(millis * 1000000);
// we must check the interrupted status in case this is used in a loop
// Otherwise we may end up spinning 100% without breaking out on an interruption
if (Thread.currentThread().isInterrupted()) {
throw new UncheckedInterruptedException();
}
}
public static void uncheckedSleepNanos(long nanos) {
LockSupport.parkNanos(nanos);
// we must check the interrupted status in case this is used in a loop
// Otherwise we may end up spinning 100% without breaking out on an interruption
if (Thread.currentThread().isInterrupted()) {
throw new UncheckedInterruptedException();
}
}
/**
* Similar to {@link InterruptedException} in concept, but unchecked. Allowing this to be thrown without being
* explicitly declared in the API.
*/
public static class UncheckedInterruptedException extends RuntimeException {
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/UIDProvider.java
+104
View File
@@ -0,0 +1,104 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.util;
import lombok.extern.slf4j.Slf4j;
import java.net.NetworkInterface;
import java.rmi.server.UID;
import java.util.Enumeration;
@Slf4j
public class UIDProvider {
private static final String JVM_UID;
private static final String HARDWARE_UID;
static {
UID jvmUIDSource = new UID();
String asString = jvmUIDSource.toString();
//Format here: hexStringFromRandomNumber:hexStringFromSystemClock:hexStringOfUIDInstance
//The first two components here will be identical for all UID instances in a JVM, where as the 'hexStringOfUIDInstance'
// will vary (increment) between UID object instances. So we'll only be using the first two components here
int lastIdx = asString.lastIndexOf(":");
JVM_UID = asString.substring(0, lastIdx).replaceAll(":", "");
//Assumptions here:
//1. getNetworkInterfaces() returns at least one non-null element
// This is guaranteed by getNetworkInterfaces() Javadoc: "The {@code Enumeration} contains at least one element..."
//2. That the iteration order for network interfaces is consistent between JVM instances on the same hardware
// This appears to hold, but no formal guarantees seem to be available here
//3. That MAC addresses are 'unique enough' for our purposes
byte[] address = null;
boolean noInterfaces = false;
Enumeration<NetworkInterface> niEnumeration = null;
try {
niEnumeration = NetworkInterface.getNetworkInterfaces();
} catch (Exception e) {
noInterfaces = true;
}
if (niEnumeration != null) {
while (niEnumeration.hasMoreElements()) {
NetworkInterface ni = niEnumeration.nextElement();
byte[] addr;
try {
addr = ni.getHardwareAddress();
} catch (Exception e) {
continue;
}
if (addr == null || addr.length != 6)
continue; //May be null (if it can't be obtained) or not standard 6 byte MAC-48 representation
address = addr;
break;
}
}
if (address == null) {
log.warn("Could not generate hardware UID{}. Using fallback: JVM UID as hardware UID.",
(noInterfaces ? " (no interfaces)" : ""));
HARDWARE_UID = JVM_UID;
} else {
StringBuilder sb = new StringBuilder();
for (byte b : address) {
sb.append(String.format("%02x", b));
}
HARDWARE_UID = sb.toString();
}
}
private UIDProvider() {}
public static String getJVMUID() {
return JVM_UID;
}
public static String getHardwareUID() {
return HARDWARE_UID;
}
}
deeplearning4j/deeplearning4j-core/src/main/java9/module-info.java
+30
View File
@@ -0,0 +1,30 @@
open module deeplearning4j.core {
requires commons.codec;
requires commons.io;
requires deeplearning4j.datavec.iterators;
requires deeplearning4j.modelimport;
requires deeplearning4j.ui.components;
requires jackson;
requires java.desktop;
requires java.rmi;
requires oshi.core;
requires resources;
requires slf4j.api;
requires datavec.api;
requires deeplearning4j.nn;
requires nd4j.api;
requires nd4j.common;
requires oshi.json;
exports org.deeplearning4j.core.datasets.test;
exports org.deeplearning4j.core.datasets.vectorizer;
exports org.deeplearning4j.core.evaluation;
exports org.deeplearning4j.core.listener;
exports org.deeplearning4j.core.loader;
exports org.deeplearning4j.core.loader.impl;
exports org.deeplearning4j.core.parallelism;
exports org.deeplearning4j.core.storage;
exports org.deeplearning4j.core.storage.impl;
exports org.deeplearning4j.core.storage.listener;
exports org.deeplearning4j.core.ui;
exports org.deeplearning4j.core.util;
}
deeplearning4j/deeplearning4j-core/src/main/resources/iris.dat
Executable
+150
View File
@@ -0,0 +1,150 @@
5.1,3.5,1.4,0.2,0
4.9,3.0,1.4,0.2,0
4.7,3.2,1.3,0.2,0
4.6,3.1,1.5,0.2,0
5.0,3.6,1.4,0.2,0
5.4,3.9,1.7,0.4,0
4.6,3.4,1.4,0.3,0
5.0,3.4,1.5,0.2,0
4.4,2.9,1.4,0.2,0
4.9,3.1,1.5,0.1,0
5.4,3.7,1.5,0.2,0
4.8,3.4,1.6,0.2,0
4.8,3.0,1.4,0.1,0
4.3,3.0,1.1,0.1,0
5.8,4.0,1.2,0.2,0
5.7,4.4,1.5,0.4,0
5.4,3.9,1.3,0.4,0
5.1,3.5,1.4,0.3,0
5.7,3.8,1.7,0.3,0
5.1,3.8,1.5,0.3,0
5.4,3.4,1.7,0.2,0
5.1,3.7,1.5,0.4,0
4.6,3.6,1.0,0.2,0
5.1,3.3,1.7,0.5,0
4.8,3.4,1.9,0.2,0
5.0,3.0,1.6,0.2,0
5.0,3.4,1.6,0.4,0
5.2,3.5,1.5,0.2,0
5.2,3.4,1.4,0.2,0
4.7,3.2,1.6,0.2,0
4.8,3.1,1.6,0.2,0
5.4,3.4,1.5,0.4,0
5.2,4.1,1.5,0.1,0
5.5,4.2,1.4,0.2,0
4.9,3.1,1.5,0.1,0
5.0,3.2,1.2,0.2,0
5.5,3.5,1.3,0.2,0
4.9,3.1,1.5,0.1,0
4.4,3.0,1.3,0.2,0
5.1,3.4,1.5,0.2,0
5.0,3.5,1.3,0.3,0
4.5,2.3,1.3,0.3,0
4.4,3.2,1.3,0.2,0
5.0,3.5,1.6,0.6,0
5.1,3.8,1.9,0.4,0
4.8,3.0,1.4,0.3,0
5.1,3.8,1.6,0.2,0
4.6,3.2,1.4,0.2,0
5.3,3.7,1.5,0.2,0
5.0,3.3,1.4,0.2,0
7.0,3.2,4.7,1.4,1
6.4,3.2,4.5,1.5,1
6.9,3.1,4.9,1.5,1
5.5,2.3,4.0,1.3,1
6.5,2.8,4.6,1.5,1
5.7,2.8,4.5,1.3,1
6.3,3.3,4.7,1.6,1
4.9,2.4,3.3,1.0,1
6.6,2.9,4.6,1.3,1
5.2,2.7,3.9,1.4,1
5.0,2.0,3.5,1.0,1
5.9,3.0,4.2,1.5,1
6.0,2.2,4.0,1.0,1
6.1,2.9,4.7,1.4,1
5.6,2.9,3.6,1.3,1
6.7,3.1,4.4,1.4,1
5.6,3.0,4.5,1.5,1
5.8,2.7,4.1,1.0,1
6.2,2.2,4.5,1.5,1
5.6,2.5,3.9,1.1,1
5.9,3.2,4.8,1.8,1
6.1,2.8,4.0,1.3,1
6.3,2.5,4.9,1.5,1
6.1,2.8,4.7,1.2,1
6.4,2.9,4.3,1.3,1
6.6,3.0,4.4,1.4,1
6.8,2.8,4.8,1.4,1
6.7,3.0,5.0,1.7,1
6.0,2.9,4.5,1.5,1
5.7,2.6,3.5,1.0,1
5.5,2.4,3.8,1.1,1
5.5,2.4,3.7,1.0,1
5.8,2.7,3.9,1.2,1
6.0,2.7,5.1,1.6,1
5.4,3.0,4.5,1.5,1
6.0,3.4,4.5,1.6,1
6.7,3.1,4.7,1.5,1
6.3,2.3,4.4,1.3,1
5.6,3.0,4.1,1.3,1
5.5,2.5,4.0,1.3,1
5.5,2.6,4.4,1.2,1
6.1,3.0,4.6,1.4,1
5.8,2.6,4.0,1.2,1
5.0,2.3,3.3,1.0,1
5.6,2.7,4.2,1.3,1
5.7,3.0,4.2,1.2,1
5.7,2.9,4.2,1.3,1
6.2,2.9,4.3,1.3,1
5.1,2.5,3.0,1.1,1
5.7,2.8,4.1,1.3,1
6.3,3.3,6.0,2.5,2
5.8,2.7,5.1,1.9,2
7.1,3.0,5.9,2.1,2
6.3,2.9,5.6,1.8,2
6.5,3.0,5.8,2.2,2
7.6,3.0,6.6,2.1,2
4.9,2.5,4.5,1.7,2
7.3,2.9,6.3,1.8,2
6.7,2.5,5.8,1.8,2
7.2,3.6,6.1,2.5,2
6.5,3.2,5.1,2.0,2
6.4,2.7,5.3,1.9,2
6.8,3.0,5.5,2.1,2
5.7,2.5,5.0,2.0,2
5.8,2.8,5.1,2.4,2
6.4,3.2,5.3,2.3,2
6.5,3.0,5.5,1.8,2
7.7,3.8,6.7,2.2,2
7.7,2.6,6.9,2.3,2
6.0,2.2,5.0,1.5,2
6.9,3.2,5.7,2.3,2
5.6,2.8,4.9,2.0,2
7.7,2.8,6.7,2.0,2
6.3,2.7,4.9,1.8,2
6.7,3.3,5.7,2.1,2
7.2,3.2,6.0,1.8,2
6.2,2.8,4.8,1.8,2
6.1,3.0,4.9,1.8,2
6.4,2.8,5.6,2.1,2
7.2,3.0,5.8,1.6,2
7.4,2.8,6.1,1.9,2
7.9,3.8,6.4,2.0,2
6.4,2.8,5.6,2.2,2
6.3,2.8,5.1,1.5,2
6.1,2.6,5.6,1.4,2
7.7,3.0,6.1,2.3,2
6.3,3.4,5.6,2.4,2
6.4,3.1,5.5,1.8,2
6.0,3.0,4.8,1.8,2
6.9,3.1,5.4,2.1,2
6.7,3.1,5.6,2.4,2
6.9,3.1,5.1,2.3,2
5.8,2.7,5.1,1.9,2
6.8,3.2,5.9,2.3,2
6.7,3.3,5.7,2.5,2
6.7,3.0,5.2,2.3,2
6.3,2.5,5.0,1.9,2
6.5,3.0,5.2,2.0,2
6.2,3.4,5.4,2.3,2
5.9,3.0,5.1,1.8,2
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/pom.xml
+71
View File
@@ -0,0 +1,71 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ /* ******************************************************************************
~ *
~ *
~ * This program and the accompanying materials are made available under the
~ * terms of the Apache License, Version 2.0 which is available at
~ * https://www.apache.org/licenses/LICENSE-2.0.
~ *
~ * See the NOTICE file distributed with this work for additional
~ * information regarding copyright ownership.
~ * Unless required by applicable law or agreed to in writing, software
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
~ * License for the specific language governing permissions and limitations
~ * under the License.
~ *
~ * SPDX-License-Identifier: Apache-2.0
~ ******************************************************************************/
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>deeplearning4j-data</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>deeplearning4j-datasets</artifactId>
<packaging>jar</packaging>
<name>deeplearning4j-datasets</name>
<properties>
<module.name>deeplearning4j.datasets</module.name>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.moditect</groupId>
<artifactId>moditect-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>datavec-data-image</artifactId>
<version>${datavec.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>deeplearning4j-datavec-iterators</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>resources</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/base/EmnistFetcher.java
+127
View File
@@ -0,0 +1,127 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.base;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.deeplearning4j.resources.DataSetResource;
import org.eclipse.deeplearning4j.resources.ResourceDataSets;
import org.eclipse.deeplearning4j.resources.utils.EMnistSet;
import java.io.File;
import java.io.IOException;
@Slf4j
public class EmnistFetcher extends MnistFetcher {
private final EMnistSet ds;
@Getter
private DataSetResource emnistDataTrain;
@Getter
private DataSetResource emnistDataTest;
@Getter
private DataSetResource emnistLabelsTrain;
@Getter
private DataSetResource emnistLabelsTest;
@Getter
private DataSetResource emnistMappingTrain;
@Getter
private DataSetResource emnistMappingTest;
public EmnistFetcher() {
this(EMnistSet.MNIST);
}
public EmnistFetcher(EMnistSet ds) {
this.ds = ds;
emnistDataTrain = ResourceDataSets.emnistTrain(ds);
emnistDataTest = ResourceDataSets.emnistTest(ds);
emnistLabelsTrain = ResourceDataSets.emnistLabelsTrain(ds);
emnistLabelsTest = ResourceDataSets.emnistLabelsTest(ds);
emnistMappingTrain = ResourceDataSets.emnistMappingTrain(ds);
emnistMappingTest = ResourceDataSets.emnistMappingTest(ds);
}
public EmnistFetcher(EMnistSet ds,File topLevelDir) {
this.ds = ds;
emnistDataTrain = ResourceDataSets.emnistTrain(ds,topLevelDir);
emnistDataTest = ResourceDataSets.emnistTest(ds,topLevelDir);
emnistLabelsTrain = ResourceDataSets.emnistLabelsTrain(ds,topLevelDir);
emnistLabelsTest = ResourceDataSets.emnistLabelsTest(ds,topLevelDir);
emnistMappingTrain = ResourceDataSets.emnistMappingTrain(ds,topLevelDir);
emnistMappingTest = ResourceDataSets.emnistMappingTest(ds,topLevelDir);
}
@Override
public String getName() {
return "EMNIST";
}
// --- Train files ---
public static int numLabels(EMnistSet dataSet) {
switch (dataSet) {
case COMPLETE:
return 62;
case MERGE:
return 47;
case BALANCED:
return 47;
case LETTERS:
return 26;
case DIGITS:
return 10;
case MNIST:
return 10;
default:
throw new UnsupportedOperationException("Unknown Set: " + dataSet);
}
}
@Override
public File downloadAndUntar() throws IOException {
if (fileDir != null) {
return fileDir;
}
File baseDir = getBaseDir();
if (!(baseDir.isDirectory() || baseDir.mkdir())) {
throw new IOException("Could not mkdir " + baseDir);
}
log.info("Downloading {}...", getName());
// get features
emnistDataTrain.download(true,3,300000,30000);
emnistDataTest.download(true,3,300000,30000);
emnistLabelsTrain.download(true,3,300000,30000);
emnistLabelsTest.download(true,3,300000,30000);
emnistMappingTrain.download(false,3,300000,30000);
emnistMappingTest.download(false,3,300000,30000);
// get labels
fileDir = baseDir;
return fileDir;
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/base/IrisUtils.java
+90
View File
@@ -0,0 +1,90 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.base;
import org.apache.commons.io.IOUtils;
import org.deeplearning4j.common.resources.DL4JResources;
import org.deeplearning4j.common.resources.ResourceType;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.common.resources.Downloader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
public class IrisUtils {
private static final String IRIS_RELATIVE_URL = "datasets/iris.dat";
private static final String MD5 = "1c21400a78061197eac64c6748844216";
private IrisUtils() {}
public static List<DataSet> loadIris(int from, int to) throws IOException {
File rootDir = DL4JResources.getDirectory(ResourceType.DATASET, "iris");
File irisData = new File(rootDir, "iris.dat");
if(!irisData.exists()) {
URL url = DL4JResources.getURL(IRIS_RELATIVE_URL);
Downloader.download("Iris", url, irisData, MD5, 3);
}
@SuppressWarnings("unchecked")
List<String> lines;
try(InputStream is = new FileInputStream(irisData)){
lines = IOUtils.readLines(is);
}
List<DataSet> list = new ArrayList<>();
INDArray ret = to - from > 1 ? Nd4j.ones(Math.abs(to - from), 4) : Nd4j.ones( 4);
double[][] outcomes = new double[lines.size()][3];
int putCount = 0;
for (int i = from; i < to; i++) {
String line = lines.get(i);
String[] split = line.split(",");
addRow(ret, putCount++, split);
String outcome = split[split.length - 1];
double[] rowOutcome = new double[3];
rowOutcome[Integer.parseInt(outcome)] = 1;
outcomes[i] = rowOutcome;
}
for (int i = 0; i < ret.rows(); i++) {
DataSet add = new DataSet(ret.getRow(i, false), Nd4j.create(outcomes[from + i], 3));
list.add(add);
}
return list;
}
private static void addRow(INDArray ret, int row, String[] line) {
double[] vector = new double[4];
for (int i = 0; i < 4; i++)
vector[i] = Double.parseDouble(line[i]);
ret.putRow(row, Nd4j.create(vector));
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/base/MnistFetcher.java
+89
View File
@@ -0,0 +1,89 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.base;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.deeplearning4j.common.resources.DL4JResources;
import org.deeplearning4j.common.resources.ResourceType;
import org.eclipse.deeplearning4j.resources.DataSetResource;
import org.eclipse.deeplearning4j.resources.ResourceDataSets;
import org.nd4j.common.resources.Downloader;
import java.io.*;
import java.net.URL;
@Data
@NoArgsConstructor
@Slf4j
public class MnistFetcher {
protected static final String LOCAL_DIR_NAME = "MNIST";
protected File fileDir;
private DataSetResource mnistTrain = ResourceDataSets.mnistTrain();
private DataSetResource mnistTest = ResourceDataSets.mnistTest();
private DataSetResource mnistTrainLabels = ResourceDataSets.mnistTrainLabels();
private DataSetResource mnistTestLabels = ResourceDataSets.mnistTestLabels();
public MnistFetcher(File tempDir) {
this.fileDir = tempDir;
}
public String getName() {
return "MNIST";
}
public File getBaseDir() {
return DL4JResources.getDirectory(ResourceType.DATASET, getName());
}
public File downloadAndUntar() throws IOException {
if (fileDir != null) {
return fileDir;
}
File baseDir = getBaseDir();
if (!(baseDir.isDirectory() || baseDir.mkdir())) {
throw new IOException("Could not mkdir " + baseDir);
}
log.info("Downloading {}...", getName());
// get features
mnistTrain.download(true,3,200000,20000);
mnistTest.download(true,3,200000,20000);
mnistTrainLabels.download(true,3,200000,20000);
mnistTestLabels.download(true,3,200000,20000);
// get labels
fileDir = baseDir;
return fileDir;
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/CacheableDataSet.java
+37
View File
@@ -0,0 +1,37 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.fetchers;
import org.datavec.api.records.reader.RecordReader;
import org.datavec.image.transform.ImageTransform;
interface CacheableDataSet {
String remoteDataUrl();
String remoteDataUrl(DataSetType set);
String localCacheName();
String dataSetName(DataSetType set);
long expectedChecksum();
long expectedChecksum(DataSetType set);
boolean isCached();
RecordReader getRecordReader(long rngSeed, int[] imgDim, DataSetType set, ImageTransform imageTransform);
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/CacheableExtractableDataSetFetcher.java
+125
View File
@@ -0,0 +1,125 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.fetchers;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.deeplearning4j.common.resources.DL4JResources;
import org.deeplearning4j.common.resources.ResourceType;
import org.nd4j.common.util.ArchiveUtils;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.zip.Adler32;
import java.util.zip.Checksum;
@Slf4j
public abstract class CacheableExtractableDataSetFetcher implements CacheableDataSet {
@Override public String dataSetName(DataSetType set) { return ""; }
@Override public String remoteDataUrl() { return remoteDataUrl(DataSetType.TRAIN); }
@Override public long expectedChecksum() { return expectedChecksum(DataSetType.TRAIN); }
public void downloadAndExtract() throws IOException { downloadAndExtract(DataSetType.TRAIN); }
/**
* Downloads and extracts the local dataset.
*
* @throws IOException
*/
public void downloadAndExtract(DataSetType set) throws IOException {
String localFilename = new File(remoteDataUrl(set)).getName();
File tmpFile = new File(System.getProperty("java.io.tmpdir"), localFilename);
File localCacheDir = getLocalCacheDir();
// check empty cache
if(localCacheDir.exists()) {
File[] list = localCacheDir.listFiles();
if(list == null || list.length == 0)
localCacheDir.delete();
}
File localDestinationDir = new File(localCacheDir, dataSetName(set));
if(!localDestinationDir.exists()) {
localCacheDir.mkdirs();
tmpFile.delete();
log.info("Downloading dataset to " + tmpFile.getAbsolutePath());
FileUtils.copyURLToFile(new URL(remoteDataUrl(set)), tmpFile);
} else {
//Directory exists and is non-empty - assume OK
log.info("Using cached dataset at " + localCacheDir.getAbsolutePath());
return;
}
if(expectedChecksum(set) != 0L) {
log.info("Verifying download...");
Checksum adler = new Adler32();
FileUtils.checksum(tmpFile, adler);
long localChecksum = adler.getValue();
log.info("Checksum local is " + localChecksum + ", expecting "+expectedChecksum(set));
if(expectedChecksum(set) != localChecksum) {
log.error("Checksums do not match. Cleaning up files and failing...");
tmpFile.delete();
throw new IllegalStateException( "Dataset file failed checksum: " + tmpFile + " - expected checksum " + expectedChecksum(set)
+ " vs. actual checksum " + localChecksum + ". If this error persists, please open an issue at https://github.com/eclipse/deeplearning4j.");
}
}
try {
ArchiveUtils.unzipFileTo(tmpFile.getAbsolutePath(), localCacheDir.getAbsolutePath(), false);
} catch (Throwable t){
//Catch any errors during extraction, and delete the directory to avoid leaving the dir in an invalid state
if(localCacheDir.exists())
FileUtils.deleteDirectory(localCacheDir);
throw t;
}
}
protected File getLocalCacheDir(){
return DL4JResources.getDirectory(ResourceType.DATASET, localCacheName());
}
/**
* Returns a boolean indicating if the dataset is already cached locally.
*
* @return boolean
*/
@Override
public boolean isCached() {
return getLocalCacheDir().exists();
}
protected static void deleteIfEmpty(File localCache){
if(localCache.exists()) {
File[] files = localCache.listFiles();
if(files == null || files.length < 1){
try {
FileUtils.deleteDirectory(localCache);
} catch (IOException e){
//Ignore
log.debug("Error deleting directory: {}", localCache);
}
}
}
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/Cifar10Fetcher.java
+104
View File
@@ -0,0 +1,104 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.fetchers;
import org.datavec.api.io.filters.RandomPathFilter;
import org.datavec.api.io.labels.ParentPathLabelGenerator;
import org.datavec.api.records.reader.RecordReader;
import org.datavec.api.split.FileSplit;
import org.datavec.api.split.InputSplit;
import org.datavec.image.loader.BaseImageLoader;
import org.datavec.image.recordreader.ImageRecordReader;
import org.datavec.image.transform.ImageTransform;
import org.deeplearning4j.common.resources.DL4JResources;
import org.nd4j.common.base.Preconditions;
import java.io.File;
import java.util.Random;
public class Cifar10Fetcher extends CacheableExtractableDataSetFetcher {
public static final String LABELS_FILENAME = "labels.txt";
public static final String LOCAL_CACHE_NAME = "cifar10";
public static int INPUT_WIDTH = 32;
public static int INPUT_HEIGHT = 32;
public static int INPUT_CHANNELS = 3;
public static int NUM_LABELS = 10;
@Override
public String remoteDataUrl(DataSetType set) {
return DL4JResources.getURLString("datasets/cifar10_dl4j.v1.zip");
}
@Override
public String localCacheName(){ return LOCAL_CACHE_NAME; }
@Override
public long expectedChecksum(DataSetType set) { return 292852033L; }
@Override
public RecordReader getRecordReader(long rngSeed, int[] imgDim, DataSetType set, ImageTransform imageTransform) {
Preconditions.checkState(imgDim == null || imgDim.length == 2, "Invalid image dimensions: must be null or lenth 2. Got: %s", imgDim);
// check empty cache
File localCache = getLocalCacheDir();
deleteIfEmpty(localCache);
try {
if (!localCache.exists()){
downloadAndExtract();
}
} catch(Exception e) {
throw new RuntimeException("Could not download CIFAR-10", e);
}
Random rng = new Random(rngSeed);
File datasetPath;
switch (set) {
case TRAIN:
datasetPath = new File(localCache, "/train/");
break;
case TEST:
datasetPath = new File(localCache, "/test/");
break;
case VALIDATION:
throw new IllegalArgumentException("You will need to manually create and iterate a validation directory, CIFAR-10 does not provide labels");
default:
datasetPath = new File(localCache, "/train/");
}
// set up file paths
RandomPathFilter pathFilter = new RandomPathFilter(rng, BaseImageLoader.ALLOWED_FORMATS);
FileSplit filesInDir = new FileSplit(datasetPath, BaseImageLoader.ALLOWED_FORMATS, rng);
InputSplit[] filesInDirSplit = filesInDir.sample(pathFilter, 1);
int h = (imgDim == null ? Cifar10Fetcher.INPUT_HEIGHT : imgDim[0]);
int w = (imgDim == null ? Cifar10Fetcher.INPUT_WIDTH : imgDim[1]);
ImageRecordReader rr = new ImageRecordReader(h, w, Cifar10Fetcher.INPUT_CHANNELS, new ParentPathLabelGenerator(), imageTransform);
try {
rr.initialize(filesInDirSplit[0]);
} catch(Exception e) {
throw new RuntimeException(e);
}
return rr;
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/DataSetType.java
+25
View File
@@ -0,0 +1,25 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.fetchers;
public enum DataSetType {
TRAIN, TEST, VALIDATION
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/EmnistDataFetcher.java
+115
View File
@@ -0,0 +1,115 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.fetchers;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.deeplearning4j.datasets.base.EmnistFetcher;
import org.deeplearning4j.common.resources.DL4JResources;
import org.deeplearning4j.common.resources.ResourceType;
import org.eclipse.deeplearning4j.resources.utils.EMnistSet;
import org.deeplearning4j.datasets.iterator.impl.EmnistDataSetIterator;
import org.deeplearning4j.datasets.mnist.MnistManager;
import org.nd4j.linalg.dataset.api.iterator.fetcher.DataSetFetcher;
import java.io.File;
import java.io.IOException;
import java.util.Random;
@Slf4j
public class EmnistDataFetcher extends MnistDataFetcher implements DataSetFetcher {
protected EmnistFetcher fetcher;
public EmnistDataFetcher(EMnistSet dataSet, boolean binarize, boolean train, boolean shuffle,
long rngSeed,File topLevelDir) throws IOException {
fetcher = new EmnistFetcher(dataSet,topLevelDir);
if (!emnistExists(fetcher)) {
fetcher.downloadAndUntar();
}
String EMNIST_ROOT = topLevelDir.getAbsolutePath();
if (train) {
images = fetcher.getEmnistDataTrain().localPath().getAbsolutePath();
labels = fetcher.getEmnistLabelsTrain().localPath().getAbsolutePath();
totalExamples = EmnistDataSetIterator.numExamplesTrain(dataSet);
} else {
images = fetcher.getEmnistDataTest().localPath().getAbsolutePath();
labels = fetcher.getEmnistLabelsTest().localPath().getAbsolutePath();
totalExamples = EmnistDataSetIterator.numExamplesTest(dataSet);
}
try {
manager = new MnistManager(images, labels, totalExamples);
} catch (Exception e) {
log.error("",e);
FileUtils.deleteDirectory(new File(EMNIST_ROOT));
new EmnistFetcher(dataSet).downloadAndUntar();
manager = new MnistManager(images, labels, totalExamples);
}
numOutcomes = EmnistDataSetIterator.numLabels(dataSet);
this.binarize = binarize;
cursor = 0;
manager.setCurrent(cursor);
inputColumns = manager.getImages().getEntryLength();
this.train = train;
this.shuffle = shuffle;
order = new int[totalExamples];
for (int i = 0; i < order.length; i++)
order[i] = i;
rng = new Random(rngSeed);
reset(); //Shuffle order
//For some inexplicable reason, EMNIST LETTERS set is indexed 1 to 26 (i.e., 1 to nClasses), while everything else
// is indexed (0 to nClasses-1) :/
if (dataSet == EMnistSet.LETTERS) {
oneIndexed = true;
} else {
oneIndexed = false;
}
this.fOrder = true; //MNIST is C order, EMNIST is F order
}
public EmnistDataFetcher(EMnistSet dataSet, boolean binarize, boolean train, boolean shuffle,
long rngSeed) throws IOException {
this(dataSet,binarize,train,shuffle,rngSeed,DL4JResources.getDirectory(ResourceType.DATASET, "EMNIST"));
}
private boolean emnistExists(EmnistFetcher e) {
//Check 4 files:
if (!fetcher.getEmnistDataTrain().existsLocally())
return false;
if (!fetcher.getEmnistLabelsTrain().existsLocally())
return false;
if (!fetcher.getEmnistDataTest().existsLocally())
return false;
if (!fetcher.getEmnistLabelsTest().existsLocally())
return false;
return true;
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/IrisDataFetcher.java
Executable
+60
View File
@@ -0,0 +1,60 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.fetchers;
import org.deeplearning4j.datasets.base.IrisUtils;
import org.nd4j.linalg.dataset.api.iterator.fetcher.BaseDataFetcher;
import java.io.IOException;
public class IrisDataFetcher extends BaseDataFetcher {
/**
*
*/
private static final long serialVersionUID = 4566329799221375262L;
public final static int NUM_EXAMPLES = 150;
public IrisDataFetcher() {
numOutcomes = 3;
inputColumns = 4;
totalExamples = NUM_EXAMPLES;
}
@Override
public void fetch(int numExamples) {
int from = cursor;
int to = cursor + numExamples;
if (to > totalExamples)
to = totalExamples;
try {
initializeCurrFromList(IrisUtils.loadIris(from, to));
cursor += numExamples;
} catch (IOException e) {
throw new IllegalStateException("Unable to load iris.dat", e);
}
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/MnistDataFetcher.java
Executable
+278
View File
@@ -0,0 +1,278 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.fetchers;
import lombok.SneakyThrows;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.deeplearning4j.datasets.base.MnistFetcher;
import org.deeplearning4j.common.resources.DL4JResources;
import org.deeplearning4j.common.resources.ResourceType;
import org.deeplearning4j.datasets.mnist.MnistManager;
import org.eclipse.deeplearning4j.resources.DataSetResource;
import org.eclipse.deeplearning4j.resources.ResourceDataSets;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.iterator.fetcher.BaseDataFetcher;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.indexing.NDArrayIndex;
import org.nd4j.common.util.MathUtils;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Random;
import java.util.zip.Adler32;
import java.util.zip.Checksum;
public class MnistDataFetcher extends BaseDataFetcher {
public static final int NUM_EXAMPLES = 60000;
public static final int NUM_EXAMPLES_TEST = 10000;
protected static final long CHECKSUM_TRAIN_FEATURES = 2094436111L;
protected static final long CHECKSUM_TRAIN_LABELS = 4008842612L;
protected static final long CHECKSUM_TEST_FEATURES = 2165396896L;
protected static final long CHECKSUM_TEST_LABELS = 2212998611L;
protected static final long[] CHECKSUMS_TRAIN = new long[]{CHECKSUM_TRAIN_FEATURES, CHECKSUM_TRAIN_LABELS};
protected static final long[] CHECKSUMS_TEST = new long[]{CHECKSUM_TEST_FEATURES, CHECKSUM_TEST_LABELS};
protected boolean binarize = true;
protected boolean train;
protected int[] order;
protected Random rng;
protected boolean shuffle;
protected boolean oneIndexed = false;
protected boolean fOrder = false; //MNIST is C order, EMNIST is F order
protected boolean firstShuffle = true;
protected int numExamples = 0;
protected String images,labels;
//note: we default to zero here on purpose, otherwise when first initializes an error is thrown.
private long lastCursor = 0;
protected MnistManager manager;
/**
* Constructor telling whether to binarize the dataset or not
* @param binarize whether to binarize the dataset or not
* @throws IOException
*/
public MnistDataFetcher(boolean binarize) throws IOException {
this(binarize, true, true, System.currentTimeMillis(), NUM_EXAMPLES);
}
public MnistDataFetcher(boolean binarize, boolean train, boolean shuffle, long rngSeed, int numExamples,File topLevelDir) throws IOException {
if(this instanceof EmnistDataFetcher)
return;
this.topLevelDir = topLevelDir;
long[] checksums;
DataSetResource imageResource = null;
DataSetResource labelResource = null;
if (train) {
imageResource = topLevelDir() != null ? ResourceDataSets.mnistTrain(topLevelDir()) : ResourceDataSets.mnistTrain();
if(!imageResource.existsLocally())
imageResource.download(true,3,200000,20000);
labelResource = topLevelDir() != null ? ResourceDataSets.mnistTrainLabels(topLevelDir()) : ResourceDataSets.mnistTrainLabels();
if(!labelResource.existsLocally())
labelResource.download(true,3,200000,20000);
totalExamples = NUM_EXAMPLES;
checksums = CHECKSUMS_TRAIN;
} else {
imageResource = topLevelDir() != null ? ResourceDataSets.mnistTest(topLevelDir()) : ResourceDataSets.mnistTest();
if(!imageResource.existsLocally())
imageResource.download(true,3,200000,20000);
labelResource = topLevelDir() != null ? ResourceDataSets.mnistTestLabels(topLevelDir()) : ResourceDataSets.mnistTestLabels();
if(!labelResource.existsLocally())
labelResource.download(true,3,200000,20000);
totalExamples = NUM_EXAMPLES_TEST;
checksums = CHECKSUMS_TEST;
}
images = imageResource.localPath().getAbsolutePath();
labels = labelResource.localPath().getAbsolutePath();
try {
manager = new MnistManager(images, labels, train);
} catch (Exception e) {
throw new RuntimeException(e);
}
numOutcomes = 10;
this.binarize = binarize;
cursor = 0;
inputColumns = manager.getImages().getEntryLength();
this.train = train;
this.shuffle = shuffle;
if (train) {
order = new int[NUM_EXAMPLES];
} else {
order = new int[NUM_EXAMPLES_TEST];
}
for (int i = 0; i < order.length; i++)
order[i] = i;
rng = new Random(rngSeed);
this.numExamples = numExamples;
reset(); //Shuffle order
}
public MnistDataFetcher(boolean binarize, boolean train, boolean shuffle, long rngSeed, int numExamples) throws IOException {
this(binarize,train,shuffle,rngSeed,numExamples,null);
}
private void validateFiles(String[] files, long[] checksums) {
//Validate files:
try {
for (int i = 0; i < files.length; i++) {
File f = new File(files[i]);
Checksum adler = new Adler32();
long checksum = f.exists() ? FileUtils.checksum(f, adler).getValue() : -1;
if (!f.exists() || checksum != checksums[i]) {
throw new IllegalStateException("Failed checksum: expected " + checksums[i] +
", got " + checksum + " for file: " + f);
}
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public MnistDataFetcher() throws IOException {
this(true);
}
private float[][] featureData = null;
@SneakyThrows
@Override
public void fetch(int numExamples) {
if (!hasMore()) {
throw new IllegalStateException("Unable to get more; there are no more images");
}
manager.setCurrent((int) lastCursor);
INDArray labels = Nd4j.zeros(DataType.FLOAT, numExamples, numOutcomes);
if(featureData == null || featureData.length < numExamples){
featureData = new float[numExamples][28 * 28];
}
int actualExamples = 0;
byte[] working = null;
for (int i = 0; i < numExamples; i++, cursor++) {
if (!hasMore())
break;
manager.setCurrent(cursor);
lastCursor = cursor;
byte[] img = manager.readImageUnsafe(order[cursor]);
if (fOrder) {
//EMNIST requires F order to C order
if (working == null) {
working = new byte[28 * 28];
}
for (int j = 0; j < 28 * 28; j++) {
working[j] = img[28 * (j % 28) + j / 28];
}
img = working;
}
int label = manager.readLabel(order[cursor]);
if (oneIndexed) {
//For some inexplicable reason, Emnist LETTERS set is indexed 1 to 26 (i.e., 1 to nClasses), while everything else
// is indexed (0 to nClasses-1) :/
label--;
}
labels.put(actualExamples, label, 1.0f);
for(int j = 0 ; j < img.length ; j++) {
featureData[actualExamples][j] = ((int) img[j]) & 0xFF;
}
actualExamples++;
}
INDArray features;
if(featureData.length == actualExamples){
features = Nd4j.create(featureData);
} else {
features = Nd4j.create(Arrays.copyOfRange(featureData, 0, actualExamples));
}
if (actualExamples < numExamples) {
labels = labels.get(NDArrayIndex.interval(0, actualExamples), NDArrayIndex.all());
}
if(binarize){
features = features.gt(30.0).castTo(DataType.FLOAT);
} else {
features.divi(255.0);
}
curr = new DataSet(features, labels);
}
@Override
public void reset() {
cursor = 0;
curr = null;
if (shuffle) {
if((train && numExamples < NUM_EXAMPLES) || (!train && numExamples < NUM_EXAMPLES_TEST)){
//Shuffle only first N elements
if(firstShuffle){
MathUtils.shuffleArray(order, rng);
firstShuffle = false;
} else {
MathUtils.shuffleArraySubset(order, numExamples, rng);
}
} else {
MathUtils.shuffleArray(order, rng);
}
}
}
@Override
public DataSet next() {
DataSet next = super.next();
return next;
}
public void close() {
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcher.java
+132
View File
@@ -0,0 +1,132 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.fetchers;
import org.datavec.api.records.reader.RecordReader;
import org.datavec.api.split.FileSplit;
import org.datavec.image.loader.BaseImageLoader;
import org.datavec.image.recordreader.objdetect.ObjectDetectionRecordReader;
import org.datavec.image.transform.ImageTransform;
import java.io.File;
import java.io.IOException;
import java.util.Random;
public class SvhnDataFetcher extends CacheableExtractableDataSetFetcher {
private static String BASE_URL = "http://ufldl.stanford.edu/";
public static void setBaseUrl(String baseUrl){
BASE_URL = baseUrl;
}
public static int NUM_LABELS = 10;
@Override
public String remoteDataUrl(DataSetType set) {
switch (set) {
case TRAIN:
return BASE_URL + "housenumbers/train.tar.gz";
case TEST:
return BASE_URL + "housenumbers/test.tar.gz";
case VALIDATION:
return BASE_URL + "housenumbers/extra.tar.gz";
default:
throw new IllegalArgumentException("Unknown DataSetType:" + set);
}
}
@Override
public String localCacheName() {
return "SVHN";
}
@Override
public String dataSetName(DataSetType set) {
switch (set) {
case TRAIN:
return "train";
case TEST:
return "test";
case VALIDATION:
return "extra";
default:
throw new IllegalArgumentException("Unknown DataSetType:" + set);
}
}
@Override
public long expectedChecksum(DataSetType set) {
switch (set) {
case TRAIN:
return 979655493L;
case TEST:
return 1629515343L;
case VALIDATION:
return 132781169L;
default:
throw new IllegalArgumentException("Unknown DataSetType:" + set);
}
}
public File getDataSetPath(DataSetType set) throws IOException {
File localCache = getLocalCacheDir();
// check empty cache
deleteIfEmpty(localCache);
File datasetPath;
switch (set) {
case TRAIN:
datasetPath = new File(localCache, "/train/");
break;
case TEST:
datasetPath = new File(localCache, "/test/");
break;
case VALIDATION:
datasetPath = new File(localCache, "/extra/");
break;
default:
datasetPath = null;
}
if (!datasetPath.exists()) {
downloadAndExtract(set);
}
return datasetPath;
}
@Override
public RecordReader getRecordReader(long rngSeed, int[] imgDim, DataSetType set, ImageTransform imageTransform) {
try {
Random rng = new Random(rngSeed);
File datasetPath = getDataSetPath(set);
FileSplit data = new FileSplit(datasetPath, BaseImageLoader.ALLOWED_FORMATS, rng);
ObjectDetectionRecordReader recordReader = new ObjectDetectionRecordReader(imgDim[1], imgDim[0], imgDim[2],
imgDim[4], imgDim[3], null);
recordReader.initialize(data);
return recordReader;
} catch (IOException e) {
throw new RuntimeException("Could not download SVHN", e);
}
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/TinyImageNetFetcher.java
+105
View File
@@ -0,0 +1,105 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.fetchers;
import org.datavec.api.io.filters.RandomPathFilter;
import org.datavec.api.io.labels.ParentPathLabelGenerator;
import org.datavec.api.records.reader.RecordReader;
import org.datavec.api.split.FileSplit;
import org.datavec.api.split.InputSplit;
import org.datavec.image.loader.BaseImageLoader;
import org.datavec.image.recordreader.ImageRecordReader;
import org.datavec.image.transform.ImageTransform;
import org.deeplearning4j.common.resources.DL4JResources;
import org.nd4j.common.base.Preconditions;
import java.io.File;
import java.util.Random;
public class TinyImageNetFetcher extends CacheableExtractableDataSetFetcher {
public static final String WORDS_FILENAME = "words.txt";
public static final String LOCAL_CACHE_NAME = "TINYIMAGENET_200";
public static int INPUT_WIDTH = 64;
public static int INPUT_HEIGHT = 64;
public static int INPUT_CHANNELS = 3;
public static int NUM_LABELS = 200;
public static int NUM_EXAMPLES = NUM_LABELS*500;
@Override
public String remoteDataUrl(DataSetType set) {
return DL4JResources.getURLString("datasets/tinyimagenet_200_dl4j.v1.zip");
}
@Override
public String localCacheName(){ return LOCAL_CACHE_NAME; }
@Override
public long expectedChecksum(DataSetType set) { return 33822361L; }
@Override
public RecordReader getRecordReader(long rngSeed, int[] imgDim, DataSetType set, ImageTransform imageTransform) {
Preconditions.checkState(imgDim == null || imgDim.length == 2, "Invalid image dimensions: must be null or lenth 2. Got: %s", imgDim);
// check empty cache
File localCache = getLocalCacheDir();
deleteIfEmpty(localCache);
try {
if (!localCache.exists()){
downloadAndExtract();
}
} catch(Exception e) {
throw new RuntimeException("Could not download TinyImageNet", e);
}
Random rng = new Random(rngSeed);
File datasetPath;
switch (set) {
case TRAIN:
datasetPath = new File(localCache, "/train/");
break;
case TEST:
datasetPath = new File(localCache, "/test/");
break;
case VALIDATION:
throw new IllegalArgumentException("You will need to manually iterate the /validation/images/ directory, TinyImageNet does not provide labels");
default:
datasetPath = new File(localCache, "/train/");
}
// set up file paths
RandomPathFilter pathFilter = new RandomPathFilter(rng, BaseImageLoader.ALLOWED_FORMATS);
FileSplit filesInDir = new FileSplit(datasetPath, BaseImageLoader.ALLOWED_FORMATS, rng);
InputSplit[] filesInDirSplit = filesInDir.sample(pathFilter, 1);
int h = (imgDim == null ? TinyImageNetFetcher.INPUT_HEIGHT : imgDim[0]);
int w = (imgDim == null ? TinyImageNetFetcher.INPUT_WIDTH : imgDim[1]);
ImageRecordReader rr = new ImageRecordReader(h, w,TinyImageNetFetcher.INPUT_CHANNELS, new ParentPathLabelGenerator(), imageTransform);
try {
rr.initialize(filesInDirSplit[0]);
} catch(Exception e) {
throw new RuntimeException(e);
}
return rr;
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/UciSequenceDataFetcher.java
+164
View File
@@ -0,0 +1,164 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.fetchers;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader;
import org.datavec.api.split.NumberedFileInputSplit;
import org.datavec.image.transform.ImageTransform;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Random;
@Slf4j
public class UciSequenceDataFetcher extends CacheableExtractableDataSetFetcher {
public static int NUM_LABELS = 6;
public static int NUM_EXAMPLES = NUM_LABELS * 100;
private static String url = "https://archive.ics.uci.edu/ml/machine-learning-databases/synthetic_control-mld/synthetic_control.data";
public static void setURL(String url){
UciSequenceDataFetcher.url = url;
}
@Override
public String remoteDataUrl() {
return url;
}
@Override
public String remoteDataUrl(DataSetType type) {
return remoteDataUrl();
}
@Override
public String localCacheName() {
return "UCISequence_6";
}
@Override
public long expectedChecksum() {
return 104392751L;
}
@Override
public long expectedChecksum(DataSetType type) {
return expectedChecksum();
}
@Override
public CSVSequenceRecordReader getRecordReader(long rngSeed, int[] shape, DataSetType set, ImageTransform transform) {
return getRecordReader(rngSeed, set);
}
public CSVSequenceRecordReader getRecordReader(long rngSeed, DataSetType set) {
// check empty cache
File localCache = getLocalCacheDir();
deleteIfEmpty(localCache);
try {
if (!localCache.exists()) downloadAndExtract();
} catch (Exception e) {
throw new RuntimeException("Could not download UCI Sequence data", e);
}
File dataPath;
switch (set) {
case TRAIN:
dataPath = new File(localCache, "/train");
break;
case TEST:
dataPath = new File(localCache, "/test");
break;
case VALIDATION:
throw new IllegalArgumentException("You will need to manually iterate the directory, UCISequence data does not provide labels");
default:
dataPath = new File(localCache, "/train");
}
try {
downloadUCIData(dataPath);
CSVSequenceRecordReader data;
switch (set) {
case TRAIN:
data = new CSVSequenceRecordReader(0, ", ");
data.initialize(new NumberedFileInputSplit(dataPath.getAbsolutePath() + "/%d.csv", 0, 449));
break;
case TEST:
data = new CSVSequenceRecordReader(0, ", ");
data.initialize(new NumberedFileInputSplit(dataPath.getAbsolutePath() + "/%d.csv", 450, 599));
break;
default:
data = new CSVSequenceRecordReader(0, ", ");
data.initialize(new NumberedFileInputSplit(dataPath.getAbsolutePath() + "/%d.csv", 0, 449));
}
return data;
} catch (Exception e) {
throw new RuntimeException("Could not process UCI data", e);
}
}
private static void downloadUCIData(File dataPath) throws Exception {
//if (dataPath.exists()) return;
String data = IOUtils.toString(new URL(url), Charset.defaultCharset());
String[] lines = data.split("\n");
int lineCount = 0;
int index = 0;
ArrayList<String> linesList = new ArrayList<>();
for (String line : lines) {
// label value
int count = lineCount++ / 100;
// replace white space with commas and label value + new line
line = line.replaceAll("\\s+", ", " + count + "\n");
// add label to last number
line = line + ", " + count;
linesList.add(line);
}
// randomly shuffle data
Collections.shuffle(linesList, new Random(12345));
for (String line : linesList) {
File outPath = new File(dataPath, index + ".csv");
FileUtils.writeStringToFile(outPath, line, Charset.defaultCharset());
index++;
}
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/Cifar10DataSetIterator.java
+127
View File
@@ -0,0 +1,127 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator.impl;
import lombok.Getter;
import org.apache.commons.io.FileUtils;
import org.datavec.image.transform.ImageTransform;
import org.deeplearning4j.common.resources.DL4JResources;
import org.deeplearning4j.common.resources.ResourceType;
import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator;
import org.deeplearning4j.datasets.fetchers.Cifar10Fetcher;
import org.deeplearning4j.datasets.fetchers.DataSetType;
import org.nd4j.common.base.Preconditions;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class Cifar10DataSetIterator extends RecordReaderDataSetIterator {
@Getter
protected DataSetPreProcessor preProcessor;
/**
* Create an iterator for the training set, with random iteration order (RNG seed fixed to 123)
*
* @param batchSize Minibatch size for the iterator
*/
public Cifar10DataSetIterator(int batchSize) {
this(batchSize, null, DataSetType.TRAIN, null, 123);
}
/**
* * Create an iterator for the training or test set, with random iteration order (RNG seed fixed to 123)
*
* @param batchSize Minibatch size for the iterator
* @param set The dataset (train or test)
*/
public Cifar10DataSetIterator(int batchSize, DataSetType set) {
this(batchSize, null, set, null, 123);
}
/**
* Get the Tiny ImageNet iterator with specified train/test set, with random iteration order (RNG seed fixed to 123)
*
* @param batchSize Size of each patch
* @param imgDim Dimensions of desired output - for example, {64, 64}
* @param set Train, test, or validation
*/
public Cifar10DataSetIterator(int batchSize, int[] imgDim, DataSetType set) {
this(batchSize, imgDim, set, null, 123);
}
/**
* Get the Tiny ImageNet iterator with specified train/test set and (optional) custom transform.
*
* @param batchSize Size of each patch
* @param imgDim Dimensions of desired output - for example, {64, 64}
* @param set Train, test, or validation
* @param imageTransform Additional image transform for output
* @param rngSeed random number generator seed to use when shuffling examples
*/
public Cifar10DataSetIterator(int batchSize, int[] imgDim, DataSetType set,
ImageTransform imageTransform, long rngSeed) {
super(new Cifar10Fetcher().getRecordReader(rngSeed, imgDim, set, imageTransform), batchSize, 1, Cifar10Fetcher.NUM_LABELS);
}
/**
* Get the labels - either in "categories" (imagenet synsets format, "n01910747" or similar) or human-readable format,
* such as "jellyfish"
* @param categories If true: return category/synset format; false: return "human readable" label format
* @return Labels
*/
public static List<String> getLabels(boolean categories){
List<String> rawLabels = new Cifar10DataSetIterator(1).getLabels();
if(categories){
return rawLabels;
}
//Otherwise, convert to human-readable format, using 'words.txt' file
File baseDir = DL4JResources.getDirectory(ResourceType.DATASET, Cifar10Fetcher.LOCAL_CACHE_NAME);
File labelFile = new File(baseDir, Cifar10Fetcher.LABELS_FILENAME);
List<String> lines;
try {
lines = FileUtils.readLines(labelFile, StandardCharsets.UTF_8);
} catch (IOException e){
throw new RuntimeException("Error reading label file", e);
}
Map<String,String> map = new HashMap<>();
for(String line : lines){
String[] split = line.split("\t");
map.put(split[0], split[1]);
}
List<String> outLabels = new ArrayList<>(rawLabels.size());
for(String s : rawLabels){
String s2 = map.get(s);
Preconditions.checkState(s2 != null, "Label \"%s\" not found in labels.txt file");
outLabels.add(s2);
}
return outLabels;
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/EmnistDataSetIterator.java
+295
View File
@@ -0,0 +1,295 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator.impl;
import lombok.Getter;
import org.deeplearning4j.common.resources.DL4JResources;
import org.deeplearning4j.common.resources.ResourceType;
import org.deeplearning4j.datasets.fetchers.EmnistDataFetcher;
import org.eclipse.deeplearning4j.resources.utils.EMnistSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.BaseDatasetIterator;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class EmnistDataSetIterator extends BaseDatasetIterator {
private static final int NUM_COMPLETE_TRAIN = 697932;
private static final int NUM_COMPLETE_TEST = 116323;
private static final int NUM_MERGE_TRAIN = 697932;
private static final int NUM_MERGE_TEST = 116323;
private static final int NUM_BALANCED_TRAIN = 112800;
private static final int NUM_BALANCED_TEST = 18800;
private static final int NUM_DIGITS_TRAIN = 240000;
private static final int NUM_DIGITS_TEST = 40000;
private static final int NUM_LETTERS_TRAIN = 88800;
private static final int NUM_LETTERS_TEST = 14800;
private static final int NUM_MNIST_TRAIN = 60000;
private static final int NUM_MNIST_TEST = 10000;
private static final char[] LABELS_COMPLETE = new char[] {48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 65, 66, 67, 68,
69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99,
100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
120, 121, 122};
private static final char[] LABELS_MERGE = new char[] {48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 65, 66, 67, 68, 69,
70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 100,
101, 102, 103, 104, 110, 113, 114, 116};
private static final char[] LABELS_BALANCED = new char[] {48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 65, 66, 67, 68,
69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 100,
101, 102, 103, 104, 110, 113, 114, 116};
private static final char[] LABELS_DIGITS = new char[] {48, 49, 50, 51, 52, 53, 54, 55, 56, 57};
private static final char[] LABELS_LETTERS = new char[] {65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90};
protected EMnistSet dataSet;
protected int batch, numExamples;
@Getter
protected DataSetPreProcessor preProcessor;
/**
* Create an EMNIST iterator with randomly shuffled data based on a random RNG seed
*
* @param dataSet Dataset (subset) to return
* @param batch Batch size
* @param train If true: use training set. If false: use test set
* @throws IOException If an error occurs when loading/downloading the dataset
*/
public EmnistDataSetIterator(EMnistSet dataSet, int batch, boolean train) throws IOException {
this(dataSet, batch, train, System.currentTimeMillis());
}
/**
* Create an EMNIST iterator with randomly shuffled data based on a specified RNG seed
*
* @param dataSet Dataset (subset) to return
* @param batchSize Batch size
* @param train If true: use training set. If false: use test set
* @param seed Random number generator seed
*/
public EmnistDataSetIterator(EMnistSet dataSet, int batchSize, boolean train, long seed) throws IOException {
this(dataSet, batchSize, false, train, true, seed);
}
/**
* Get the specified number of MNIST examples (test or train set), with optional shuffling and binarization.
*
* @param batch Size of each minibatch
* @param binarize whether to binarize the data or not (if false: normalize in range 0 to 1)
* @param train Train vs. test set
* @param shuffle whether to shuffle the examples
* @param rngSeed random number generator seed to use when shuffling examples
*/
public EmnistDataSetIterator(EMnistSet dataSet, int batch, boolean binarize, boolean train, boolean shuffle, long rngSeed, File topLevelDir)
throws IOException {
super(batch, numExamples(train, dataSet), new EmnistDataFetcher(dataSet, binarize, train, shuffle, rngSeed));
this.dataSet = dataSet;
}
/**
* Get the specified number of MNIST examples (test or train set), with optional shuffling and binarization.
*
* @param batch Size of each minibatch
* @param binarize whether to binarize the data or not (if false: normalize in range 0 to 1)
* @param train Train vs. test set
* @param shuffle whether to shuffle the examples
* @param rngSeed random number generator seed to use when shuffling examples
*/
public EmnistDataSetIterator(EMnistSet dataSet, int batch, boolean binarize, boolean train, boolean shuffle, long rngSeed)
throws IOException {
this(dataSet,batch,binarize,train,shuffle,rngSeed, DL4JResources.getDirectory(ResourceType.DATASET,"emnist"));
}
private static int numExamples(boolean train, EMnistSet ds) {
if (train) {
return numExamplesTrain(ds);
} else {
return numExamplesTest(ds);
}
}
/**
* Get the number of training examples for the specified subset
*
* @param dataSet Subset to get
* @return Number of examples for the specified subset
*/
public static int numExamplesTrain(EMnistSet dataSet) {
switch (dataSet) {
case COMPLETE:
return NUM_COMPLETE_TRAIN;
case MERGE:
return NUM_MERGE_TRAIN;
case BALANCED:
return NUM_BALANCED_TRAIN;
case LETTERS:
return NUM_LETTERS_TRAIN;
case DIGITS:
return NUM_DIGITS_TRAIN;
case MNIST:
return NUM_MNIST_TRAIN;
default:
throw new UnsupportedOperationException("Unknown Set: " + dataSet);
}
}
/**
* Get the number of test examples for the specified subset
*
* @param dataSet Subset to get
* @return Number of examples for the specified subset
*/
public static int numExamplesTest(EMnistSet dataSet) {
switch (dataSet) {
case COMPLETE:
return NUM_COMPLETE_TEST;
case MERGE:
return NUM_MERGE_TEST;
case BALANCED:
return NUM_BALANCED_TEST;
case LETTERS:
return NUM_LETTERS_TEST;
case DIGITS:
return NUM_DIGITS_TEST;
case MNIST:
return NUM_MNIST_TEST;
default:
throw new UnsupportedOperationException("Unknown Set: " + dataSet);
}
}
/**
* Get the number of labels for the specified subset
*
* @param dataSet Subset to get
* @return Number of labels for the specified subset
*/
public static int numLabels(EMnistSet dataSet) {
switch (dataSet) {
case COMPLETE:
return 62;
case MERGE:
return 47;
case BALANCED:
return 47;
case LETTERS:
return 26;
case DIGITS:
return 10;
case MNIST:
return 10;
default:
throw new UnsupportedOperationException("Unknown Set: " + dataSet);
}
}
/**
* Get the labels as a character array
*
* @return Labels
*/
public char[] getLabelsArrays() {
return getLabelsArray(dataSet);
}
/**
* Get the labels as a List<String>
*
* @return Labels
*/
public List<String> getLabels() {
return getLabels(dataSet);
}
/**
* Get the label assignments for the given set as a character array.
*
* @param dataSet DataSet to get the label assignment for
* @return Label assignment and given dataset
*/
public static char[] getLabelsArray(EMnistSet dataSet) {
switch (dataSet) {
case COMPLETE:
return LABELS_COMPLETE;
case MERGE:
return LABELS_MERGE;
case BALANCED:
return LABELS_BALANCED;
case LETTERS:
return LABELS_LETTERS;
case DIGITS:
case MNIST:
return LABELS_DIGITS;
default:
throw new UnsupportedOperationException("Unknown Set: " + dataSet);
}
}
/**
* Get the label assignments for the given set as a List<String>
*
* @param dataSet DataSet to get the label assignment for
* @return Label assignment and given dataset
*/
public static List<String> getLabels(EMnistSet dataSet) {
char[] c = getLabelsArray(dataSet);
List<String> l = new ArrayList<>(c.length);
for (char c2 : c) {
l.add(String.valueOf(c2));
}
return l;
}
/**
* Are the labels balanced in the training set (that is: are the number of examples for each label equal?)
*
* @param dataSet Set to get balanced value for
* @return True if balanced dataset, false otherwise
*/
public static boolean isBalanced(EMnistSet dataSet) {
switch (dataSet) {
case COMPLETE:
case MERGE:
case LETTERS:
//Note: EMNIST docs claims letters is balanced, but this is not possible for training set:
// 88800 examples / 26 classes = 3418.46
return false;
case BALANCED:
case DIGITS:
case MNIST:
return true;
default:
throw new UnsupportedOperationException("Unknown Set: " + dataSet);
}
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/IrisDataSetIterator.java
Executable
+63
View File
@@ -0,0 +1,63 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator.impl;
import org.deeplearning4j.datasets.fetchers.IrisDataFetcher;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.iterator.BaseDatasetIterator;
public class IrisDataSetIterator extends BaseDatasetIterator {
/**
*
*/
private static final long serialVersionUID = -2022454995728680368L;
/**
* Create an iris iterator for full batch training - i.e., all 150 examples are included per minibatch
*/
public IrisDataSetIterator(){
this(150, 150);
}
/**
* IrisDataSetIterator handles traversing through the Iris Data Set.
* @see <a href="https://archive.ics.uci.edu/ml/datasets/Iris">https://archive.ics.uci.edu/ml/datasets/Iris</a>
*
* @param batch Batch size
* @param numExamples Total number of examples
*/
public IrisDataSetIterator(int batch, int numExamples) {
super(batch, numExamples, new IrisDataFetcher());
}
@Override
public DataSet next() {
fetcher.fetch(batch);
DataSet next = fetcher.next();
if(preProcessor != null) {
preProcessor.preProcess(next);
}
return next;
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/LFWDataSetIterator.java
Executable
+106
View File
@@ -0,0 +1,106 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator.impl;
import org.datavec.api.io.labels.ParentPathLabelGenerator;
import org.datavec.api.io.labels.PathLabelGenerator;
import org.datavec.image.loader.LFWLoader;
import org.datavec.image.transform.ImageTransform;
import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator;
import java.util.Random;
public class LFWDataSetIterator extends RecordReaderDataSetIterator {
/** Loads subset of images with given imgDim returned by the generator. */
public LFWDataSetIterator(int[] imgDim) {
this(LFWLoader.SUB_NUM_IMAGES, LFWLoader.SUB_NUM_IMAGES, imgDim, LFWLoader.SUB_NUM_LABELS, false,
new ParentPathLabelGenerator(), true, 1, null, new Random(System.currentTimeMillis()));
}
/** Loads images with given batchSize, numExamples returned by the generator. */
public LFWDataSetIterator(int batchSize, int numExamples) {
this(batchSize, numExamples, new int[] {LFWLoader.HEIGHT, LFWLoader.WIDTH, LFWLoader.CHANNELS},
LFWLoader.NUM_LABELS, false, LFWLoader.LABEL_PATTERN, true, 1, null,
new Random(System.currentTimeMillis()));
}
/** Loads images with given batchSize, numExamples, imgDim returned by the generator. */
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim) {
this(batchSize, numExamples, imgDim, LFWLoader.NUM_LABELS, false, LFWLoader.LABEL_PATTERN, true, 1, null,
new Random(System.currentTimeMillis()));
}
/** Loads images with given batchSize, imgDim, useSubset, returned by the generator. */
public LFWDataSetIterator(int batchSize, int[] imgDim, boolean useSubset) {
this(batchSize, useSubset ? LFWLoader.SUB_NUM_IMAGES : LFWLoader.NUM_IMAGES, imgDim,
useSubset ? LFWLoader.SUB_NUM_LABELS : LFWLoader.NUM_LABELS, useSubset, LFWLoader.LABEL_PATTERN,
true, 1, null, new Random(System.currentTimeMillis()));
}
/** Loads images with given batchSize, numExamples, imgDim, train, & splitTrainTest returned by the generator. */
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, boolean train, double splitTrainTest) {
this(batchSize, numExamples, imgDim, LFWLoader.NUM_LABELS, false, LFWLoader.LABEL_PATTERN, train,
splitTrainTest, null, new Random(System.currentTimeMillis()));
}
/** Loads images with given batchSize, numExamples, numLabels, train, & splitTrainTest returned by the generator. */
public LFWDataSetIterator(int batchSize, int numExamples, int numLabels, boolean train, double splitTrainTest) {
this(batchSize, numExamples, new int[] {LFWLoader.HEIGHT, LFWLoader.WIDTH, LFWLoader.CHANNELS}, numLabels,
false, null, train, splitTrainTest, null, new Random(System.currentTimeMillis()));
}
/** Loads images with given batchSize, numExamples, imgDim, numLabels, useSubset, train, splitTrainTest & Random returned by the generator. */
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset,
boolean train, double splitTrainTest, Random rng) {
this(batchSize, numExamples, imgDim, numLabels, useSubset, LFWLoader.LABEL_PATTERN, train, splitTrainTest, null,
rng);
}
/** Loads images with given batchSize, numExamples, imgDim, numLabels, useSubset, train, splitTrainTest & Random returned by the generator. */
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset,
PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
this(batchSize, numExamples, imgDim, numLabels, useSubset, labelGenerator, train, splitTrainTest, null, rng);
}
/**
* Create LFW data specific iterator
* @param batchSize the batch size of the examples
* @param numExamples the overall number of examples
* @param imgDim an array of height, width and channels
* @param numLabels the overall number of examples
* @param useSubset use a subset of the LFWDataSet
* @param labelGenerator path label generator to use
* @param train true if use train value
* @param splitTrainTest the percentage to split data for train and remainder goes to test
* @param imageTransform how to transform the image
* @param rng random number to lock in batch shuffling
* */
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset,
PathLabelGenerator labelGenerator, boolean train, double splitTrainTest,
ImageTransform imageTransform, Random rng) {
super(new LFWLoader(imgDim, imageTransform, useSubset).getRecordReader(batchSize, numExamples, imgDim,
numLabels, labelGenerator, train, splitTrainTest, rng), batchSize, 1, numLabels);
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/MnistDataSetIterator.java
Executable
+89
View File
@@ -0,0 +1,89 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator.impl;
import org.deeplearning4j.datasets.fetchers.MnistDataFetcher;
import org.nd4j.linalg.dataset.api.iterator.BaseDatasetIterator;
import java.io.File;
import java.io.IOException;
public class MnistDataSetIterator extends BaseDatasetIterator {
public MnistDataSetIterator(int batch, int numExamples) throws IOException {
this(batch, numExamples, false);
}
/**Get the specified number of examples for the MNIST training data set.
* @param batch the batch size of the examples
* @param numExamples the overall number of examples
* @param binarize whether to binarize mnist or not
* @throws IOException
*/
public MnistDataSetIterator(int batch, int numExamples, boolean binarize) throws IOException {
this(batch, numExamples, binarize, true, false, 0);
}
/** Constructor to get the full MNIST data set (either test or train sets) without binarization (i.e., just normalization
* into range of 0 to 1), with shuffling based on a random seed.
* @param batchSize
* @param train
* @throws IOException
*/
public MnistDataSetIterator(int batchSize, boolean train, int seed) throws IOException {
this(batchSize, (train ? MnistDataFetcher.NUM_EXAMPLES : MnistDataFetcher.NUM_EXAMPLES_TEST), false, train,
true, seed);
}
/**Get the specified number of MNIST examples (test or train set), with optional shuffling and binarization.
* @param batch Size of each patch
* @param numExamples total number of examples to load
* @param binarize whether to binarize the data or not (if false: normalize in range 0 to 1)
* @param train Train vs. test set
* @param shuffle whether to shuffle the examples
* @param rngSeed random number generator seed to use when shuffling examples
*/
public MnistDataSetIterator(int batch, int numExamples, boolean binarize, boolean train, boolean shuffle,
long rngSeed) throws IOException {
this(batch, numExamples, binarize,train,shuffle,rngSeed,null);
}
/**Get the specified number of MNIST examples (test or train set), with optional shuffling and binarization.
* @param batch Size of each patch
* @param numExamples total number of examples to load
* @param binarize whether to binarize the data or not (if false: normalize in range 0 to 1)
* @param train Train vs. test set
* @param shuffle whether to shuffle the examples
* @param rngSeed random number generator seed to use when shuffling examples
*/
public MnistDataSetIterator(int batch, int numExamples, boolean binarize, boolean train, boolean shuffle,
long rngSeed, File topLevelDir) throws IOException {
super(batch, numExamples, new MnistDataFetcher(binarize, train, shuffle, rngSeed, numExamples,topLevelDir));
}
public void close() {
MnistDataFetcher mnistDataFetcher = (MnistDataFetcher) fetcher;
mnistDataFetcher.close();
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/TinyImageNetDataSetIterator.java
+127
View File
@@ -0,0 +1,127 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator.impl;
import lombok.Getter;
import org.apache.commons.io.FileUtils;
import org.datavec.image.transform.ImageTransform;
import org.deeplearning4j.common.resources.DL4JResources;
import org.deeplearning4j.common.resources.ResourceType;
import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator;
import org.deeplearning4j.datasets.fetchers.DataSetType;
import org.deeplearning4j.datasets.fetchers.TinyImageNetFetcher;
import org.nd4j.common.base.Preconditions;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class TinyImageNetDataSetIterator extends RecordReaderDataSetIterator {
@Getter
protected DataSetPreProcessor preProcessor;
/**
* Create an iterator for the training set, with random iteration order (RNG seed fixed to 123)
*
* @param batchSize Minibatch size for the iterator
*/
public TinyImageNetDataSetIterator(int batchSize) {
this(batchSize, null, DataSetType.TRAIN, null, 123);
}
/**
* * Create an iterator for the training or test set, with random iteration order (RNG seed fixed to 123)
*
* @param batchSize Minibatch size for the iterator
* @param set The dataset (train or test)
*/
public TinyImageNetDataSetIterator(int batchSize, DataSetType set) {
this(batchSize, null, set, null, 123);
}
/**
* Get the Tiny ImageNet iterator with specified train/test set, with random iteration order (RNG seed fixed to 123)
*
* @param batchSize Size of each patch
* @param imgDim Dimensions of desired output - for example, {64, 64}
* @param set Train, test, or validation
*/
public TinyImageNetDataSetIterator(int batchSize, int[] imgDim, DataSetType set) {
this(batchSize, imgDim, set, null, 123);
}
/**
* Get the Tiny ImageNet iterator with specified train/test set and (optional) custom transform.
*
* @param batchSize Size of each patch
* @param imgDim Dimensions of desired output - for example, {64, 64}
* @param set Train, test, or validation
* @param imageTransform Additional image transform for output
* @param rngSeed random number generator seed to use when shuffling examples
*/
public TinyImageNetDataSetIterator(int batchSize, int[] imgDim, DataSetType set,
ImageTransform imageTransform, long rngSeed) {
super(new TinyImageNetFetcher().getRecordReader(rngSeed, imgDim, set, imageTransform), batchSize, 1, TinyImageNetFetcher.NUM_LABELS);
}
/**
* Get the labels - either in "categories" (imagenet synsets format, "n01910747" or similar) or human-readable format,
* such as "jellyfish"
* @param categories If true: return category/synset format; false: return "human readable" label format
* @return Labels
*/
public static List<String> getLabels(boolean categories){
List<String> rawLabels = new TinyImageNetDataSetIterator(1).getLabels();
if(categories){
return rawLabels;
}
//Otherwise, convert to human-readable format, using 'words.txt' file
File baseDir = DL4JResources.getDirectory(ResourceType.DATASET, TinyImageNetFetcher.LOCAL_CACHE_NAME);
File labelFile = new File(baseDir, TinyImageNetFetcher.WORDS_FILENAME);
List<String> lines;
try {
lines = FileUtils.readLines(labelFile, StandardCharsets.UTF_8);
} catch (IOException e){
throw new RuntimeException("Error reading label file", e);
}
Map<String,String> map = new HashMap<>();
for(String line : lines){
String[] split = line.split("\t");
map.put(split[0], split[1]);
}
List<String> outLabels = new ArrayList<>(rawLabels.size());
for(String s : rawLabels){
String s2 = map.get(s);
Preconditions.checkState(s2 != null, "Label \"%s\" not found in labels.txt file");
outLabels.add(s2);
}
return outLabels;
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/UciSequenceDataSetIterator.java
+62
View File
@@ -0,0 +1,62 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator.impl;
import org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator;
import org.deeplearning4j.datasets.fetchers.DataSetType;
import org.deeplearning4j.datasets.fetchers.UciSequenceDataFetcher;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
public class UciSequenceDataSetIterator extends SequenceRecordReaderDataSetIterator {
protected DataSetPreProcessor preProcessor;
/**
* Create an iterator for the training set, with the specified minibatch size. Randomized with RNG seed 123
*
* @param batchSize Minibatch size
*/
public UciSequenceDataSetIterator(int batchSize) {
this(batchSize, DataSetType.TRAIN, 123);
}
/**
* Create an iterator for the training or test set, with the specified minibatch size. Randomized with RNG seed 123
*
* @param batchSize Minibatch size
* @param set Set: training or test
*/
public UciSequenceDataSetIterator(int batchSize, DataSetType set) {
this(batchSize, set, 123);
}
/**
* Create an iterator for the training or test set, with the specified minibatch size
*
* @param batchSize Minibatch size
* @param set Set: training or test
* @param rngSeed Random number generator seed to use for randomization
*/
public UciSequenceDataSetIterator(int batchSize, DataSetType set, long rngSeed) {
super(new UciSequenceDataFetcher().getRecordReader(rngSeed, set), batchSize, UciSequenceDataFetcher.NUM_LABELS, 1);
// last parameter is index of label
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistDbFile.java
Executable
+125
View File
@@ -0,0 +1,125 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.mnist;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
public abstract class MnistDbFile extends RandomAccessFile {
private int count;
/**
* Creates new instance and reads the header information.
*
* @param name
* the system-dependent filename
* @param mode
* the access mode
* @throws IOException
* @throws FileNotFoundException
* @see RandomAccessFile
*/
public MnistDbFile(String name, String mode) throws IOException {
super(name, mode);
if (getMagicNumber() != readInt()) {
throw new RuntimeException(
"This MNIST DB file " + name + " should start with the number " + getMagicNumber() + ".");
}
count = readInt();
}
/**
* MNIST DB files start with unique integer number.
*
* @return integer number that should be found in the beginning of the file.
*/
protected abstract int getMagicNumber();
/**
* The current entry index.
*
* @return long
* @throws IOException
*/
public long getCurrentIndex() throws IOException {
return (getFilePointer() - getHeaderSize()) / getEntryLength() + 1;
}
/**
* Set the required current entry index.
*
* @param curr
* the entry index
*/
public void setCurrentIndex(long curr) {
try {
if (curr < 0 || curr > count) {
throw new RuntimeException(curr + " is not in the range 0 to " + count);
}
seek(getHeaderSize() + curr * getEntryLength());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public int getHeaderSize() {
return 8; // two integers
}
/**
* Number of bytes for each entry.
* Defaults to 1.
*
* @return int
*/
public int getEntryLength() {
return 1;
}
/**
* Move to the next entry.
*
* @throws IOException
*/
public void next() throws IOException {
if (getCurrentIndex() < count) {
skipBytes(getEntryLength());
}
}
/**
* Move to the previous entry.
*
* @throws IOException
*/
public void prev() throws IOException {
if (getCurrentIndex() > 0) {
seek(getFilePointer() - getEntryLength());
}
}
public int getCount() {
return count;
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistImageFile.java
Executable
+130
View File
@@ -0,0 +1,130 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.mnist;
import java.io.FileNotFoundException;
import java.io.IOException;
public class MnistImageFile extends MnistDbFile {
private int rows;
private int cols;
/**
* Creates new MNIST database image file ready for reading.
*
* @param name
* the system-dependent filename
* @param mode
* the access mode
* @throws IOException
* @throws FileNotFoundException
*/
public MnistImageFile(String name, String mode) throws IOException {
super(name, mode);
// read header information
rows = readInt();
cols = readInt();
}
/**
* Reads the image at the current position.
*
* @return matrix representing the image
* @throws IOException
*/
public int[][] readImage() throws IOException {
int[][] dat = new int[getRows()][getCols()];
for (int i = 0; i < getCols(); i++) {
for (int j = 0; j < getRows(); j++) {
dat[i][j] = readUnsignedByte();
}
}
return dat;
}
/** Read the specified number of images from the current position, to a byte[nImages][rows*cols]
* Note that MNIST data set is stored as unsigned bytes; this method returns signed bytes without conversion
* (i.e., same bits, but requires conversion before use)
* @param nImages Number of images
*/
public byte[][] readImagesUnsafe(int nImages) throws IOException {
byte[][] out = new byte[nImages][0];
for (int i = 0; i < nImages; i++) {
out[i] = new byte[rows * cols];
read(out[i]);
}
return out;
}
/**
* Move the cursor to the next image.
*
* @throws IOException
*/
public void nextImage() throws IOException {
super.next();
}
/**
* Move the cursor to the previous image.
*
* @throws IOException
*/
public void prevImage() throws IOException {
super.prev();
}
@Override
protected int getMagicNumber() {
return 2051;
}
/**
* Number of rows per image.
*
* @return int
*/
public int getRows() {
return rows;
}
/**
* Number of columns per image.
*
* @return int
*/
public int getCols() {
return cols;
}
@Override
public int getEntryLength() {
return cols * rows;
}
@Override
public int getHeaderSize() {
return super.getHeaderSize() + 8; // to more integers - rows and columns
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistLabelFile.java
Executable
+71
View File
@@ -0,0 +1,71 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.mnist;
import java.io.FileNotFoundException;
import java.io.IOException;
/**
*
* MNIST database label file.
*
*/
public class MnistLabelFile extends MnistDbFile {
/**
* Creates new MNIST database label file ready for reading.
*
* @param name
* the system-dependent filename
* @param mode
* the access mode
* @throws IOException
* @throws FileNotFoundException
*/
public MnistLabelFile(String name, String mode) throws IOException {
super(name, mode);
}
/**
* Reads the integer at the current position.
*
* @return integer representing the label
* @throws IOException
*/
public int readLabel() throws IOException {
return readUnsignedByte();
}
/** Read the specified number of labels from the current position*/
public int[] readLabels(int num) throws IOException {
int[] out = new int[num];
for (int i = 0; i < num; i++)
out[i] = readLabel();
return out;
}
@Override
protected int getMagicNumber() {
return 2049;
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistManager.java
Executable
+188
View File
@@ -0,0 +1,188 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.mnist;
import lombok.SneakyThrows;
import org.deeplearning4j.datasets.fetchers.MnistDataFetcher;
import org.nd4j.common.base.Preconditions;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
public class MnistManager {
MnistImageFile images;
private MnistLabelFile labels;
private byte[][] imagesArr;
private int[] labelsArr;
private static final int HEADER_SIZE = 8;
/**
* Writes the given image in the given file using the PPM data format.
*
* @param image
* @param ppmFileName
* @throws IOException
*/
public static void writeImageToPpm(int[][] image, String ppmFileName) throws IOException {
try (BufferedWriter ppmOut = new BufferedWriter(new FileWriter(ppmFileName))) {
int rows = image.length;
int cols = image[0].length;
ppmOut.write("P3\n");
ppmOut.write("" + rows + " " + cols + " 255\n");
for (int i = 0; i < rows; i++) {
StringBuilder s = new StringBuilder();
for (int j = 0; j < cols; j++) {
s.append(image[i][j] + " " + image[i][j] + " " + image[i][j] + " ");
}
ppmOut.write(s.toString());
}
}
}
@SneakyThrows
public long getCurrent() {
return labels.getCurrentIndex();
}
/**
* Constructs an instance managing the two given data files. Supports
* <code>NULL</code> value for one of the arguments in case reading only one
* of the files (images and labels) is required.
*
* @param imagesFile
* Can be <code>NULL</code>. In that case all future operations
* using that file will fail.
* @param labelsFile
* Can be <code>NULL</code>. In that case all future operations
* using that file will fail.
* @throws IOException
*/
public MnistManager(String imagesFile, String labelsFile, boolean train) throws IOException {
this(imagesFile, labelsFile, train ? MnistDataFetcher.NUM_EXAMPLES : MnistDataFetcher.NUM_EXAMPLES_TEST);
}
public MnistManager(String imagesFile, String labelsFile, int numExamples) throws IOException {
if (imagesFile != null) {
images = new MnistImageFile(imagesFile, "r");
imagesArr = images.readImagesUnsafe(numExamples);
}
if (labelsFile != null) {
labels = new MnistLabelFile(labelsFile, "r");
labelsArr = labels.readLabels(numExamples);
}
}
public MnistManager(String imagesFile, String labelsFile) throws IOException {
this(imagesFile, labelsFile, true);
}
/**
* Reads the current image.
*
* @return matrix
* @throws IOException
*/
public int[][] readImage() throws IOException {
if (images == null) {
throw new IllegalStateException("Images file not initialized.");
}
return images.readImage();
}
public byte[] readImageUnsafe(int i) {
Preconditions.checkArgument(i < imagesArr.length);
return imagesArr[i];
}
/**
* Set the position to be read.
*
* @param index
*/
public void setCurrent(int index) {
images.setCurrentIndex(index);
labels.setCurrentIndex(index);
}
/**
* Reads the current label.
*
* @return int
* @throws IOException
*/
public int readLabel() throws IOException {
if (labels == null) {
throw new IllegalStateException("labels file not initialized.");
}
return labels.readLabel();
}
public int readLabel(int i) {
return labelsArr[i];
}
/**
* Get the underlying images file as {@link MnistImageFile}.
*
* @return {@link MnistImageFile}.
*/
public MnistImageFile getImages() {
return images;
}
/**
* Get the underlying labels file as {@link MnistLabelFile}.
*
* @return {@link MnistLabelFile}.
*/
public MnistLabelFile getLabels() {
return labels;
}
/**
* Close any resources opened by the manager.
*/
public void close() {
if (images != null) {
try {
images.close();
} catch (IOException e) {
}
images = null;
}
if (labels != null) {
try {
labels.close();
} catch (IOException e) {
}
labels = null;
}
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java9/module-info.java
+15
View File
@@ -0,0 +1,15 @@
open module deeplearning4j.datasets {
requires commons.io;
requires lombok;
requires nd4j.common;
requires slf4j.api;
requires datavec.api;
requires datavec.data.image;
requires deeplearning4j.datavec.iterators;
requires nd4j.api;
requires resources;
exports org.deeplearning4j.datasets.base;
exports org.deeplearning4j.datasets.fetchers;
exports org.deeplearning4j.datasets.iterator.impl;
exports org.deeplearning4j.datasets.mnist;
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/pom.xml
+66
View File
@@ -0,0 +1,66 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ /* ******************************************************************************
~ *
~ *
~ * This program and the accompanying materials are made available under the
~ * terms of the Apache License, Version 2.0 which is available at
~ * https://www.apache.org/licenses/LICENSE-2.0.
~ *
~ * See the NOTICE file distributed with this work for additional
~ * information regarding copyright ownership.
~ * Unless required by applicable law or agreed to in writing, software
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
~ * License for the specific language governing permissions and limitations
~ * under the License.
~ *
~ * SPDX-License-Identifier: Apache-2.0
~ ******************************************************************************/
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>deeplearning4j-data</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>deeplearning4j-datavec-iterators</artifactId>
<packaging>jar</packaging>
<name>deeplearning4j-datavec-iterators</name>
<properties>
<module.name>deeplearning4j.datavec.iterators</module.name>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.moditect</groupId>
<artifactId>moditect-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>datavec-api</artifactId>
<version>${datavec.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>nd4j-api</artifactId>
</dependency>
</dependencies>
</project>
deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/RecordReaderDataSetIterator.java
+580
View File
@@ -0,0 +1,580 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.datavec;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.datavec.api.io.WritableConverter;
import org.datavec.api.io.converters.SelfWritableConverter;
import org.datavec.api.records.Record;
import org.datavec.api.records.metadata.RecordMetaData;
import org.datavec.api.records.metadata.RecordMetaDataComposableMap;
import org.datavec.api.records.reader.RecordReader;
import org.datavec.api.records.reader.SequenceRecordReader;
import org.datavec.api.records.reader.impl.ConcatenatingRecordReader;
import org.datavec.api.records.reader.impl.collection.CollectionRecordReader;
import org.datavec.api.writable.Writable;
import org.nd4j.common.base.Preconditions;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import org.nd4j.linalg.dataset.api.MultiDataSet;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.factory.Nd4j;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@Slf4j
public class RecordReaderDataSetIterator implements DataSetIterator {
private static final String READER_KEY = "reader";
@Getter
protected RecordReader recordReader;
protected WritableConverter converter;
protected int batchSize = 10;
protected int maxNumBatches = -1;
protected int batchNum = 0;
protected int labelIndex = -1;
protected int labelIndexTo = -1;
protected int numPossibleLabels = -1;
protected Iterator<List<Writable>> sequenceIter;
protected DataSet last;
protected boolean useCurrent = false;
protected boolean regression = false;
@Getter
protected DataSetPreProcessor preProcessor;
@Getter
private boolean collectMetaData = false;
private RecordReaderMultiDataSetIterator underlying;
private boolean underlyingIsDisjoint;
/**
* Constructor for classification, where:<br>
* (a) the label index is assumed to be the very last Writable/column, and<br>
* (b) the number of classes is inferred from RecordReader.getLabels()<br>
* Note that if RecordReader.getLabels() returns null, no output labels will be produced
*
* @param recordReader Record reader to use as the source of data
* @param batchSize Minibatch size, for each call of .next()
*/
public RecordReaderDataSetIterator(RecordReader recordReader, int batchSize) {
this(recordReader, new SelfWritableConverter(), batchSize, -1, -1,
recordReader.getLabels() == null ? -1 : recordReader.getLabels().size(), -1, false);
}
/**
* Main constructor for classification. This will convert the input class index (at position labelIndex, with integer
* values 0 to numPossibleLabels-1 inclusive) to the appropriate one-hot output/labels representation.
*
* @param recordReader RecordReader: provides the source of the data
* @param batchSize Batch size (number of examples) for the output DataSet objects
* @param labelIndex Index of the label Writable (usually an IntWritable), as obtained by recordReader.next()
* @param numPossibleLabels Number of classes (possible labels) for classification
*/
public RecordReaderDataSetIterator(RecordReader recordReader, int batchSize, int labelIndex,
int numPossibleLabels) {
this(recordReader, new SelfWritableConverter(), batchSize, labelIndex, labelIndex, numPossibleLabels, -1, false);
}
/**
* Constructor for classification, where the maximum number of returned batches is limited to the specified value
*
* @param recordReader the recordreader to use
* @param labelIndex the index/column of the label (for classification)
* @param numPossibleLabels the number of possible labels for classification. Not used if regression == true
* @param maxNumBatches The maximum number of batches to return between resets. Set to -1 to return all available data
*/
public RecordReaderDataSetIterator(RecordReader recordReader, int batchSize, int labelIndex, int numPossibleLabels,
int maxNumBatches) {
this(recordReader, new SelfWritableConverter(), batchSize, labelIndex, labelIndex, numPossibleLabels, maxNumBatches, false);
}
/**
* Main constructor for multi-label regression (i.e., regression with multiple outputs). Can also be used for single
* output regression with labelIndexFrom == labelIndexTo
*
* @param recordReader RecordReader to get data from
* @param labelIndexFrom Index of the first regression target
* @param labelIndexTo Index of the last regression target, inclusive
* @param batchSize Minibatch size
* @param regression Require regression = true. Mainly included to avoid clashing with other constructors previously defined :/
*/
public RecordReaderDataSetIterator(RecordReader recordReader, int batchSize, int labelIndexFrom, int labelIndexTo,
boolean regression) {
this(recordReader, new SelfWritableConverter(), batchSize, labelIndexFrom, labelIndexTo, -1, -1, regression);
if (!regression) {
throw new IllegalArgumentException("This constructor is only for creating regression iterators. " +
"If you're doing classification you need to use another constructor that " +
"(implicitly) specifies numPossibleLabels");
}
}
/**
* Main constructor
*
* @param recordReader the recordreader to use
* @param converter Converter. May be null.
* @param batchSize Minibatch size - number of examples returned for each call of .next()
* @param labelIndexFrom the index of the label (for classification), or the first index of the labels for multi-output regression
* @param labelIndexTo only used if regression == true. The last index <i>inclusive</i> of the multi-output regression
* @param numPossibleLabels the number of possible labels for classification. Not used if regression == true
* @param maxNumBatches Maximum number of batches to return
* @param regression if true: regression. If false: classification (assume labelIndexFrom is the class it belongs to)
*/
public RecordReaderDataSetIterator(RecordReader recordReader, WritableConverter converter, int batchSize,
int labelIndexFrom, int labelIndexTo, int numPossibleLabels, int maxNumBatches,
boolean regression) {
this.recordReader = recordReader;
this.converter = converter;
this.batchSize = batchSize;
this.maxNumBatches = maxNumBatches;
this.labelIndex = labelIndexFrom;
this.labelIndexTo = labelIndexTo;
this.numPossibleLabels = numPossibleLabels;
this.regression = regression;
}
protected RecordReaderDataSetIterator(Builder b){
this.recordReader = b.recordReader;
this.converter = b.converter;
this.batchSize = b.batchSize;
this.maxNumBatches = b.maxNumBatches;
this.labelIndex = b.labelIndex;
this.labelIndexTo = b.labelIndexTo;
this.numPossibleLabels = b.numPossibleLabels;
this.regression = b.regression;
this.preProcessor = b.preProcessor;
this.collectMetaData = b.collectMetaData;
}
/**
* When set to true: metadata for the current examples will be present in the returned DataSet.
* Disabled by default.
*
* @param collectMetaData Whether to collect metadata or not
*/
public void setCollectMetaData(boolean collectMetaData) {
if (underlying != null) {
underlying.setCollectMetaData(collectMetaData);
}
this.collectMetaData = collectMetaData;
}
private void initializeUnderlying() {
if (underlying == null) {
Record next = recordReader.nextRecord();
initializeUnderlying(next);
}
}
private void initializeUnderlying(Record next) {
int totalSize = next.getRecord().size();
//allow people to specify label index as -1 and infer the last possible label
if (numPossibleLabels >= 1 && labelIndex < 0) {
labelIndex = totalSize - 1;
labelIndexTo = labelIndex;
}
if(recordReader.resetSupported()) {
recordReader.reset();
} else {
//Hack around the fact that we need the first record to initialize the underlying RRMDSI, but can't reset
// the original reader
recordReader = new ConcatenatingRecordReader(
new CollectionRecordReader(Collections.singletonList(next.getRecord())),
recordReader);
}
RecordReaderMultiDataSetIterator.Builder builder = new RecordReaderMultiDataSetIterator.Builder(batchSize);
if (recordReader instanceof SequenceRecordReader) {
builder.addSequenceReader(READER_KEY, (SequenceRecordReader) recordReader);
} else {
builder.addReader(READER_KEY, recordReader);
}
if (regression) {
builder.addOutput(READER_KEY, labelIndex, labelIndexTo);
} else if (numPossibleLabels >= 1) {
builder.addOutputOneHot(READER_KEY, labelIndex, numPossibleLabels);
}
//Inputs: assume to be all the other writables
//In general: can't assume label indices are all at the start or end (event though 99% of the time they are)
//If they are: easy. If not: use 2 inputs in the underlying as a workaround, and concat them
if (labelIndex >= 0 && (labelIndex == 0 || labelIndexTo == totalSize - 1)) {
//Labels are first or last -> one input in underlying
int inputFrom;
int inputTo;
if (labelIndex < 0) {
//No label
inputFrom = 0;
inputTo = totalSize - 1;
} else if (labelIndex == 0) {
inputFrom = labelIndexTo + 1;
inputTo = totalSize - 1;
} else {
inputFrom = 0;
inputTo = labelIndex - 1;
}
builder.addInput(READER_KEY, inputFrom, inputTo);
underlyingIsDisjoint = false;
} else if (labelIndex >= 0) {
Preconditions.checkState(labelIndex < next.getRecord().size(),
"Invalid label (from) index: index must be in range 0 to first record size of (0 to %s inclusive), got %s", next.getRecord().size()-1, labelIndex);
Preconditions.checkState(labelIndexTo < next.getRecord().size(),
"Invalid label (to) index: index must be in range 0 to first record size of (0 to %s inclusive), got %s", next.getRecord().size()-1, labelIndexTo);
//Multiple inputs
int firstFrom = 0;
int firstTo = labelIndex - 1;
int secondFrom = labelIndexTo + 1;
int secondTo = totalSize - 1;
builder.addInput(READER_KEY, firstFrom, firstTo);
builder.addInput(READER_KEY, secondFrom, secondTo);
underlyingIsDisjoint = true;
} else {
//No labels - only features
builder.addInput(READER_KEY);
underlyingIsDisjoint = false;
}
underlying = builder.build();
if (collectMetaData) {
underlying.setCollectMetaData(true);
}
}
private DataSet mdsToDataSet(MultiDataSet mds) {
INDArray f;
INDArray fm;
if (underlyingIsDisjoint) {
//Rare case: 2 input arrays -> concat
INDArray f1 = getOrNull(mds.getFeatures(), 0);
INDArray f2 = getOrNull(mds.getFeatures(), 1);
fm = getOrNull(mds.getFeaturesMaskArrays(), 0); //Per-example masking only on the input -> same for both
//Can assume 2d features here
f = Nd4j.hstack(f1, f2);
} else {
//Standard case
f = getOrNull(mds.getFeatures(), 0);
fm = getOrNull(mds.getFeaturesMaskArrays(), 0);
}
INDArray l = getOrNull(mds.getLabels(), 0);
INDArray lm = getOrNull(mds.getLabelsMaskArrays(), 0);
DataSet ds = new DataSet(f, l, fm, lm);
if (collectMetaData) {
List<Serializable> temp = mds.getExampleMetaData();
List<Serializable> temp2 = new ArrayList<>(temp.size());
for (Serializable s : temp) {
RecordMetaDataComposableMap m = (RecordMetaDataComposableMap) s;
temp2.add(m.getMeta().get(READER_KEY));
}
ds.setExampleMetaData(temp2);
}
//Edge case, for backward compatibility:
//If labelIdx == -1 && numPossibleLabels == -1 -> no labels -> set labels array to features array
if (labelIndex == -1 && numPossibleLabels == -1 && ds.getLabels() == null) {
ds.setLabels(ds.getFeatures());
}
if (preProcessor != null) {
preProcessor.preProcess(ds);
}
return ds;
}
@Override
public DataSet next(int num) {
if (useCurrent) {
useCurrent = false;
if (preProcessor != null)
preProcessor.preProcess(last);
return last;
}
if (underlying == null) {
initializeUnderlying();
}
batchNum++;
return mdsToDataSet(underlying.next(num));
}
//Package private
static INDArray getOrNull(INDArray[] arr, int idx) {
if (arr == null || arr.length == 0) {
return null;
}
return arr[idx];
}
@Override
public int inputColumns() {
if (last == null) {
DataSet next = next();
last = next;
useCurrent = true;
return next.numInputs();
} else
return last.numInputs();
}
@Override
public int totalOutcomes() {
if (last == null) {
DataSet next = next();
last = next;
useCurrent = true;
return next.numOutcomes();
} else
return last.numOutcomes();
}
@Override
public boolean resetSupported() {
if(underlying == null){
initializeUnderlying();
}
return underlying.resetSupported();
}
@Override
public boolean asyncSupported() {
return true;
}
@Override
public void reset() {
batchNum = 0;
if (underlying != null) {
underlying.reset();
}
last = null;
useCurrent = false;
}
@Override
public int batch() {
return batchSize;
}
@Override
public void setPreProcessor(org.nd4j.linalg.dataset.api.DataSetPreProcessor preProcessor) {
this.preProcessor = preProcessor;
}
@Override
public boolean hasNext() {
return (((sequenceIter != null && sequenceIter.hasNext()) || recordReader.hasNext())
&& (maxNumBatches < 0 || batchNum < maxNumBatches));
}
@Override
public DataSet next() {
return next(batchSize);
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
@Override
public List<String> getLabels() {
return recordReader.getLabels();
}
/**
* Load a single example to a DataSet, using the provided RecordMetaData.
* Note that it is more efficient to load multiple instances at once, using {@link #loadFromMetaData(List)}
*
* @param recordMetaData RecordMetaData to load from. Should have been produced by the given record reader
* @return DataSet with the specified example
* @throws IOException If an error occurs during loading of the data
*/
public DataSet loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
return loadFromMetaData(Collections.singletonList(recordMetaData));
}
/**
* Load a multiple examples to a DataSet, using the provided RecordMetaData instances.
*
* @param list List of RecordMetaData instances to load from. Should have been produced by the record reader provided
* to the RecordReaderDataSetIterator constructor
* @return DataSet with the specified examples
* @throws IOException If an error occurs during loading of the data
*/
public DataSet loadFromMetaData(List<RecordMetaData> list) throws IOException {
if (underlying == null) {
Record r = recordReader.loadFromMetaData(list.get(0));
initializeUnderlying(r);
}
//Convert back to composable:
List<RecordMetaData> l = new ArrayList<>(list.size());
for (RecordMetaData m : list) {
l.add(new RecordMetaDataComposableMap(Collections.singletonMap(READER_KEY, m)));
}
MultiDataSet m = underlying.loadFromMetaData(l);
return mdsToDataSet(m);
}
/**
* Builder class for RecordReaderDataSetIterator
*/
public static class Builder {
protected RecordReader recordReader;
protected WritableConverter converter;
protected int batchSize;
protected int maxNumBatches = -1;
protected int labelIndex = -1;
protected int labelIndexTo = -1;
protected int numPossibleLabels = -1;
protected boolean regression = false;
protected DataSetPreProcessor preProcessor;
private boolean collectMetaData = false;
private boolean clOrRegCalled = false;
/**
*
* @param rr Underlying record reader to source data from
* @param batchSize Batch size to use
*/
public Builder(@NonNull RecordReader rr, int batchSize){
this.recordReader = rr;
this.batchSize = batchSize;
}
public Builder writableConverter(WritableConverter converter){
this.converter = converter;
return this;
}
/**
* Optional argument, usually not used. If set, can be used to limit the maximum number of minibatches that
* will be returned (between resets). If not set, will always return as many minibatches as there is data
* available.
*
* @param maxNumBatches Maximum number of minibatches per epoch / reset
*/
public Builder maxNumBatches(int maxNumBatches){
this.maxNumBatches = maxNumBatches;
return this;
}
/**
* Use this for single output regression (i.e., 1 output/regression target)
*
* @param labelIndex Column index that contains the regression target (indexes start at 0)
*/
public Builder regression(int labelIndex){
return regression(labelIndex, labelIndex);
}
/**
* Use this for multiple output regression (1 or more output/regression targets). Note that all regression
* targets must be contiguous (i.e., positions x to y, without gaps)
*
* @param labelIndexFrom Column index of the first regression target (indexes start at 0)
* @param labelIndexTo Column index of the last regression target (inclusive)
*/
public Builder regression(int labelIndexFrom, int labelIndexTo){
this.labelIndex = labelIndexFrom;
this.labelIndexTo = labelIndexTo;
this.regression = true;
clOrRegCalled = true;
return this;
}
/**
* Use this for classification
*
* @param labelIndex Index that contains the label index. Column (indexes start from 0) be an integer value,
* and contain values 0 to numClasses-1
* @param numClasses Number of label classes (i.e., number of categories/classes in the dataset)
*/
public Builder classification(int labelIndex, int numClasses){
this.labelIndex = labelIndex;
this.labelIndexTo = labelIndex;
this.numPossibleLabels = numClasses;
this.regression = false;
clOrRegCalled = true;
return this;
}
/**
* Optional arg. Allows the preprocessor to be set
* @param preProcessor Preprocessor to use
*/
public Builder preProcessor(DataSetPreProcessor preProcessor){
this.preProcessor = preProcessor;
return this;
}
/**
* When set to true: metadata for the current examples will be present in the returned DataSet.
* Disabled by default.
*
* @param collectMetaData Whether metadata should be collected or not
*/
public Builder collectMetaData(boolean collectMetaData){
this.collectMetaData = collectMetaData;
return this;
}
public RecordReaderDataSetIterator build(){
return new RecordReaderDataSetIterator(this);
}
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/RecordReaderMultiDataSetIterator.java
+1022
View File
File diff suppressed because it is too large Load Diff
deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/SequenceRecordReaderDataSetIterator.java
+480
View File
@@ -0,0 +1,480 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.datavec;
import lombok.Getter;
import lombok.Setter;
import org.datavec.api.records.SequenceRecord;
import org.datavec.api.records.metadata.RecordMetaData;
import org.datavec.api.records.metadata.RecordMetaDataComposable;
import org.datavec.api.records.metadata.RecordMetaDataComposableMap;
import org.datavec.api.records.reader.SequenceRecordReader;
import org.deeplearning4j.datasets.datavec.exception.ZeroLengthSequenceException;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import org.nd4j.linalg.dataset.api.MultiDataSet;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.indexing.INDArrayIndex;
import org.nd4j.linalg.indexing.NDArrayIndex;
import java.io.IOException;
import java.io.Serializable;
import java.util.*;
public class SequenceRecordReaderDataSetIterator implements DataSetIterator {
/**Alignment mode for dealing with input/labels of differing lengths (for example, one-to-many and many-to-one type situations).
* For example, might have 10 time steps total but only one label at end for sequence classification.<br>
* Currently supported modes:<br>
* <b>EQUAL_LENGTH</b>: Default. Assume that label and input time series are of equal length, and all examples are of
* the same length<br>
* <b>ALIGN_START</b>: Align the label/input time series at the first time step, and zero pad either the labels or
* the input at the end<br>
* <b>ALIGN_END</b>: Align the label/input at the last time step, zero padding either the input or the labels as required<br>
*
* Note 1: When the time series for each example are of different lengths, the shorter time series will be padded to
* the length of the longest time series.<br>
* Note 2: When ALIGN_START or ALIGN_END are used, the DataSet masking functionality is used. Thus, the returned DataSets
* will have the input and mask arrays set. These mask arrays identify whether an input/label is actually present,
* or whether the value is merely masked.<br>
*/
public enum AlignmentMode {
EQUAL_LENGTH, ALIGN_START, ALIGN_END
}
private static final String READER_KEY = "reader";
private static final String READER_KEY_LABEL = "reader_labels";
private SequenceRecordReader recordReader;
private SequenceRecordReader labelsReader;
private int miniBatchSize = 10;
private final boolean regression;
private int labelIndex = -1;
private final int numPossibleLabels;
private int cursor = 0;
private int inputColumns = -1;
private int totalOutcomes = -1;
private boolean useStored = false;
private DataSet stored = null;
@Getter
private DataSetPreProcessor preProcessor;
private AlignmentMode alignmentMode;
private final boolean singleSequenceReaderMode;
@Getter
@Setter
private boolean collectMetaData = false;
private RecordReaderMultiDataSetIterator underlying;
private boolean underlyingIsDisjoint;
/**
* Constructor where features and labels come from different RecordReaders (for example, different files),
* and labels are for classification.
*
* @param featuresReader SequenceRecordReader for the features
* @param labels Labels: assume single value per time step, where values are integers in the range 0 to numPossibleLables-1
* @param miniBatchSize Minibatch size for each call of next()
* @param numPossibleLabels Number of classes for the labels
*/
public SequenceRecordReaderDataSetIterator(SequenceRecordReader featuresReader, SequenceRecordReader labels,
int miniBatchSize, int numPossibleLabels) {
this(featuresReader, labels, miniBatchSize, numPossibleLabels, false);
}
/**
* Constructor where features and labels come from different RecordReaders (for example, different files)
*/
public SequenceRecordReaderDataSetIterator(SequenceRecordReader featuresReader, SequenceRecordReader labels,
int miniBatchSize, int numPossibleLabels, boolean regression) {
this(featuresReader, labels, miniBatchSize, numPossibleLabels, regression, AlignmentMode.EQUAL_LENGTH);
}
/**
* Constructor where features and labels come from different RecordReaders (for example, different files)
*/
public SequenceRecordReaderDataSetIterator(SequenceRecordReader featuresReader, SequenceRecordReader labels,
int miniBatchSize, int numPossibleLabels, boolean regression, AlignmentMode alignmentMode) {
this.recordReader = featuresReader;
this.labelsReader = labels;
this.miniBatchSize = miniBatchSize;
this.numPossibleLabels = numPossibleLabels;
this.regression = regression;
this.alignmentMode = alignmentMode;
this.singleSequenceReaderMode = false;
}
/** Constructor where features and labels come from the SAME RecordReader (i.e., target/label is a column in the
* same data as the features). Defaults to regression = false - i.e., for classification
* @param reader SequenceRecordReader with data
* @param miniBatchSize size of each minibatch
* @param numPossibleLabels number of labels/classes for classification
* @param labelIndex index in input of the label index. If in regression mode and numPossibleLabels > 1, labelIndex denotes the
* first index for labels. Everything before that index will be treated as input(s) and
* everything from that index (inclusive) to the end will be treated as output(s)
*/
public SequenceRecordReaderDataSetIterator(SequenceRecordReader reader, int miniBatchSize, int numPossibleLabels,
int labelIndex) {
this(reader, miniBatchSize, numPossibleLabels, labelIndex, false);
}
/** Constructor where features and labels come from the SAME RecordReader (i.e., target/label is a column in the
* same data as the features)
* @param reader SequenceRecordReader with data
* @param miniBatchSize size of each minibatch
* @param numPossibleLabels number of labels/classes for classification
* @param labelIndex index in input of the label index. If in regression mode and numPossibleLabels > 1, labelIndex denotes the
* first index for labels. Everything before that index will be treated as input(s) and
* everything from that index (inclusive) to the end will be treated as output(s)
* @param regression Whether output is for regression or classification
*/
public SequenceRecordReaderDataSetIterator(SequenceRecordReader reader, int miniBatchSize, int numPossibleLabels,
int labelIndex, boolean regression) {
this.recordReader = reader;
this.labelsReader = null;
this.miniBatchSize = miniBatchSize;
this.regression = regression;
this.labelIndex = labelIndex;
this.numPossibleLabels = numPossibleLabels;
this.singleSequenceReaderMode = true;
}
private void initializeUnderlyingFromReader() {
initializeUnderlying(recordReader.nextSequence());
underlying.reset();
}
private void initializeUnderlying(SequenceRecord nextF) {
if (nextF.getSequenceRecord().isEmpty()) {
throw new ZeroLengthSequenceException();
}
int totalSizeF = nextF.getSequenceRecord().get(0).size();
//allow people to specify label index as -1 and infer the last possible label
if (singleSequenceReaderMode && numPossibleLabels >= 1 && labelIndex < 0) {
labelIndex = totalSizeF - 1;
} else if (!singleSequenceReaderMode && numPossibleLabels >= 1 && labelIndex < 0) {
labelIndex = 0;
}
recordReader.reset();
//Add readers
RecordReaderMultiDataSetIterator.Builder builder = new RecordReaderMultiDataSetIterator.Builder(miniBatchSize);
builder.addSequenceReader(READER_KEY, recordReader);
if (labelsReader != null) {
builder.addSequenceReader(READER_KEY_LABEL, labelsReader);
}
//Add outputs
if (singleSequenceReaderMode) {
if (labelIndex < 0 && numPossibleLabels < 0) {
//No labels - all values -> features array
builder.addInput(READER_KEY);
} else if (labelIndex == 0 || labelIndex == totalSizeF - 1) { //Features: subset of columns
//Labels are first or last -> one input in underlying
int inputFrom;
int inputTo;
if (labelIndex < 0) {
//No label
inputFrom = 0;
inputTo = totalSizeF - 1;
} else if (labelIndex == 0) {
inputFrom = 1;
inputTo = totalSizeF - 1;
} else {
inputFrom = 0;
inputTo = labelIndex - 1;
}
builder.addInput(READER_KEY, inputFrom, inputTo);
underlyingIsDisjoint = false;
} else if (regression && numPossibleLabels > 1){
//Multiple inputs and multiple outputs
int inputFrom = 0;
int inputTo = labelIndex - 1;
int outputFrom = labelIndex;
int outputTo = totalSizeF - 1;
builder.addInput(READER_KEY, inputFrom, inputTo);
builder.addOutput(READER_KEY, outputFrom, outputTo);
underlyingIsDisjoint = false;
} else {
//Multiple inputs (disjoint features case)
int firstFrom = 0;
int firstTo = labelIndex - 1;
int secondFrom = labelIndex + 1;
int secondTo = totalSizeF - 1;
builder.addInput(READER_KEY, firstFrom, firstTo);
builder.addInput(READER_KEY, secondFrom, secondTo);
underlyingIsDisjoint = true;
}
if(!(labelIndex < 0 && numPossibleLabels < 0)) {
if (regression && numPossibleLabels <= 1) {
//Multiple output regression already handled
builder.addOutput(READER_KEY, labelIndex, labelIndex);
} else if (!regression) {
builder.addOutputOneHot(READER_KEY, labelIndex, numPossibleLabels);
}
}
} else {
//Features: entire reader
builder.addInput(READER_KEY);
underlyingIsDisjoint = false;
if (regression) {
builder.addOutput(READER_KEY_LABEL);
} else {
builder.addOutputOneHot(READER_KEY_LABEL, 0, numPossibleLabels);
}
}
if (alignmentMode != null) {
switch (alignmentMode) {
case EQUAL_LENGTH:
builder.sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.EQUAL_LENGTH);
break;
case ALIGN_START:
builder.sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.ALIGN_START);
break;
case ALIGN_END:
builder.sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.ALIGN_END);
break;
}
}
underlying = builder.build();
if (collectMetaData) {
underlying.setCollectMetaData(true);
}
}
private DataSet mdsToDataSet(MultiDataSet mds) {
INDArray f;
INDArray fm;
if (underlyingIsDisjoint) {
//Rare case: 2 input arrays -> concat
INDArray f1 = RecordReaderDataSetIterator.getOrNull(mds.getFeatures(), 0);
INDArray f2 = RecordReaderDataSetIterator.getOrNull(mds.getFeatures(), 1);
fm = RecordReaderDataSetIterator.getOrNull(mds.getFeaturesMaskArrays(), 0); //Per-example masking only on the input -> same for both
//Can assume 3d features here
f = Nd4j.createUninitialized(new long[] {f1.size(0), f1.size(1) + f2.size(1), f1.size(2)});
f.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.interval(0, f1.size(1)), NDArrayIndex.all()},
f1);
f.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.interval(f1.size(1), f1.size(1) + f2.size(1)),
NDArrayIndex.all()}, f2);
} else {
//Standard case
f = RecordReaderDataSetIterator.getOrNull(mds.getFeatures(), 0);
fm = RecordReaderDataSetIterator.getOrNull(mds.getFeaturesMaskArrays(), 0);
}
INDArray l = RecordReaderDataSetIterator.getOrNull(mds.getLabels(), 0);
INDArray lm = RecordReaderDataSetIterator.getOrNull(mds.getLabelsMaskArrays(), 0);
DataSet ds = new DataSet(f, l, fm, lm);
if (collectMetaData) {
List<Serializable> temp = mds.getExampleMetaData();
List<Serializable> temp2 = new ArrayList<>(temp.size());
for (Serializable s : temp) {
RecordMetaDataComposableMap m = (RecordMetaDataComposableMap) s;
if (singleSequenceReaderMode) {
temp2.add(m.getMeta().get(READER_KEY));
} else {
RecordMetaDataComposable c = new RecordMetaDataComposable(m.getMeta().get(READER_KEY),
m.getMeta().get(READER_KEY_LABEL));
temp2.add(c);
}
}
ds.setExampleMetaData(temp2);
}
if (preProcessor != null) {
preProcessor.preProcess(ds);
}
return ds;
}
@Override
public boolean hasNext() {
if (underlying == null) {
initializeUnderlyingFromReader();
}
return underlying.hasNext();
}
@Override
public DataSet next() {
return next(miniBatchSize);
}
@Override
public DataSet next(int num) {
if (useStored) {
useStored = false;
DataSet temp = stored;
stored = null;
if (preProcessor != null)
preProcessor.preProcess(temp);
return temp;
}
if (!hasNext())
throw new NoSuchElementException();
if (underlying == null) {
initializeUnderlyingFromReader();
}
MultiDataSet mds = underlying.next(num);
DataSet ds = mdsToDataSet(mds);
if (totalOutcomes == -1) {
inputColumns = (int) ds.getFeatures().size(1);
totalOutcomes = ds.getLabels() == null ? -1 : (int) ds.getLabels().size(1);
}
return ds;
}
@Override
public int inputColumns() {
if (inputColumns != -1)
return inputColumns;
preLoad();
return inputColumns;
}
@Override
public int totalOutcomes() {
if (totalOutcomes != -1)
return totalOutcomes;
preLoad();
return totalOutcomes;
}
private void preLoad() {
stored = next();
useStored = true;
inputColumns = (int) stored.getFeatures().size(1);
totalOutcomes = (int) stored.getLabels().size(1);
}
@Override
public boolean resetSupported() {
return true;
}
@Override
public boolean asyncSupported() {
return true;
}
@Override
public void reset() {
if (underlying != null)
underlying.reset();
cursor = 0;
stored = null;
useStored = false;
}
@Override
public int batch() {
return miniBatchSize;
}
@Override
public void setPreProcessor(DataSetPreProcessor preProcessor) {
this.preProcessor = preProcessor;
}
@Override
public List<String> getLabels() {
return null;
}
@Override
public void remove() {
throw new UnsupportedOperationException("Remove not supported for this iterator");
}
/**
* Load a single sequence example to a DataSet, using the provided RecordMetaData.
* Note that it is more efficient to load multiple instances at once, using {@link #loadFromMetaData(List)}
*
* @param recordMetaData RecordMetaData to load from. Should have been produced by the given record reader
* @return DataSet with the specified example
* @throws IOException If an error occurs during loading of the data
*/
public DataSet loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
return loadFromMetaData(Collections.singletonList(recordMetaData));
}
/**
* Load a multiple sequence examples to a DataSet, using the provided RecordMetaData instances.
*
* @param list List of RecordMetaData instances to load from. Should have been produced by the record reader provided
* to the SequenceRecordReaderDataSetIterator constructor
* @return DataSet with the specified examples
* @throws IOException If an error occurs during loading of the data
*/
public DataSet loadFromMetaData(List<RecordMetaData> list) throws IOException {
if (underlying == null) {
SequenceRecord r = recordReader.loadSequenceFromMetaData(list.get(0));
initializeUnderlying(r);
}
//Two cases: single vs. multiple reader...
List<RecordMetaData> l = new ArrayList<>(list.size());
if (singleSequenceReaderMode) {
for (RecordMetaData m : list) {
l.add(new RecordMetaDataComposableMap(Collections.singletonMap(READER_KEY, m)));
}
} else {
for (RecordMetaData m : list) {
RecordMetaDataComposable rmdc = (RecordMetaDataComposable) m;
Map<String, RecordMetaData> map = new HashMap<>(2);
map.put(READER_KEY, rmdc.getMeta()[0]);
map.put(READER_KEY_LABEL, rmdc.getMeta()[1]);
l.add(new RecordMetaDataComposableMap(map));
}
}
return mdsToDataSet(underlying.loadFromMetaData(l));
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/exception/ZeroLengthSequenceException.java
+31
View File
@@ -0,0 +1,31 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.datavec.exception;
public class ZeroLengthSequenceException extends RuntimeException {
public ZeroLengthSequenceException() {
this("");
}
public ZeroLengthSequenceException(String type) {
super(String.format("Encountered zero-length %ssequence", type.equals("") ? "" : type + " "));
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java9/module-info.java
+9
View File
@@ -0,0 +1,9 @@
open module deeplearning4j.datavec.iterators {
requires nd4j.common;
requires org.apache.commons.lang3;
requires slf4j.api;
requires datavec.api;
requires nd4j.api;
exports org.deeplearning4j.datasets.datavec;
exports org.deeplearning4j.datasets.datavec.exception;
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/pom.xml
+60
View File
@@ -0,0 +1,60 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ /* ******************************************************************************
~ *
~ *
~ * This program and the accompanying materials are made available under the
~ * terms of the Apache License, Version 2.0 which is available at
~ * https://www.apache.org/licenses/LICENSE-2.0.
~ *
~ * See the NOTICE file distributed with this work for additional
~ * information regarding copyright ownership.
~ * Unless required by applicable law or agreed to in writing, software
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
~ * License for the specific language governing permissions and limitations
~ * under the License.
~ *
~ * SPDX-License-Identifier: Apache-2.0
~ ******************************************************************************/
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>deeplearning4j-data</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>deeplearning4j-utility-iterators</artifactId>
<packaging>jar</packaging>
<name>deeplearning4j-utility-iterators</name>
<properties>
<module.name>deeplearning4j.utility.iterators</module.name>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.moditect</groupId>
<artifactId>moditect-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>nd4j-api</artifactId>
</dependency>
</dependencies>
</project>
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AbstractDataSetIterator.java
+278
View File
@@ -0,0 +1,278 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.NonNull;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.common.primitives.Pair;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.concurrent.LinkedBlockingQueue;
public abstract class AbstractDataSetIterator<T> implements DataSetIterator {
private DataSetPreProcessor preProcessor;
private transient Iterable<Pair<T, T>> iterable;
private transient Iterator<Pair<T, T>> iterator;
private final int batchSize;
// FIXME: capacity 4 is triage here, proper investigation requires
private final LinkedBlockingQueue<DataSet> queue = new LinkedBlockingQueue<>(4);
private List<String> labels;
private int numFeatures = -1;
private int numLabels = -1;
protected AbstractDataSetIterator(@NonNull Iterable<Pair<T, T>> iterable, int batchSize) {
if (batchSize < 1)
throw new IllegalStateException("batchSize can't be < 1");
this.iterable = iterable;
this.iterator = this.iterable.iterator();
this.batchSize = batchSize;
fillQueue();
}
/**
* Like the standard next method but allows a
* customizable number of examples returned
*
* @param num the number of examples
* @return the next data applyTransformToDestination
*/
@Override
public DataSet next(int num) {
throw new IllegalStateException("next(int) isn't supported for this DataSetIterator");
}
/**
* Input columns for the dataset
*
* @return
*/
@Override
public int inputColumns() {
return numFeatures;
}
/**
* The number of labels for the dataset
*
* @return
*/
@Override
public int totalOutcomes() {
return numLabels;
}
@Override
public boolean resetSupported() {
return iterable != null;
}
@Override
public boolean asyncSupported() {
return true;
}
/**
* Resets the iterator back to the beginning
*/
@Override
public void reset() {
queue.clear();
if (iterable != null)
iterator = iterable.iterator();
}
/**
* Batch size
*
* @return
*/
@Override
public int batch() {
return batchSize;
}
/**
* Set a pre processor
*
* @param preProcessor a pre processor to set
*/
@Override
public void setPreProcessor(DataSetPreProcessor preProcessor) {
this.preProcessor = preProcessor;
}
/**
* Get dataset iterator record reader labels
*/
@Override
public List<String> getLabels() {
return labels;
}
/**
* Returns {@code true} if the iteration has more elements.
* (In other words, returns {@code true} if {@link #next} would
* return an element rather than throwing an exception.)
*
* @return {@code true} if the iteration has more elements
*/
@Override
public boolean hasNext() {
fillQueue();
return !queue.isEmpty();
}
protected void fillQueue() {
if (queue.isEmpty()) {
List<INDArray> ndLabels = null;
List<INDArray> ndFeatures = null;
float[][] fLabels = null;
float[][] fFeatures = null;
double[][] dLabels = null;
double[][] dFeatures = null;
int sampleCount = 0;
for (int cnt = 0; cnt < batchSize; cnt++) {
if (iterator.hasNext()) {
Pair<T, T> pair = iterator.next();
if (numFeatures < 1) {
if (pair.getFirst() instanceof INDArray) {
numFeatures = (int) ((INDArray) pair.getFirst()).length();
numLabels = (int) ((INDArray) pair.getSecond()).length();
} else if (pair.getFirst() instanceof float[]) {
numFeatures = ((float[]) pair.getFirst()).length;
numLabels = ((float[]) pair.getSecond()).length;
} else if (pair.getFirst() instanceof double[]) {
numFeatures = ((double[]) pair.getFirst()).length;
numLabels = ((double[]) pair.getSecond()).length;
}
}
if (pair.getFirst() instanceof INDArray) {
if (ndLabels == null) {
ndLabels = new ArrayList<>();
ndFeatures = new ArrayList<>();
}
ndFeatures.add(((INDArray) pair.getFirst()));
ndLabels.add(((INDArray) pair.getSecond()));
} else if (pair.getFirst() instanceof float[]) {
if (fLabels == null) {
fLabels = new float[batchSize][];
fFeatures = new float[batchSize][];
}
fFeatures[sampleCount] = (float[]) pair.getFirst();
fLabels[sampleCount] = (float[]) pair.getSecond();
} else if (pair.getFirst() instanceof double[]) {
if (dLabels == null) {
dLabels = new double[batchSize][];
dFeatures = new double[batchSize][];
}
dFeatures[sampleCount] = (double[]) pair.getFirst();
dLabels[sampleCount] = (double[]) pair.getSecond();
}
sampleCount += 1;
} else
break;
}
if (sampleCount == batchSize) {
INDArray labels = null;
INDArray features = null;
if (ndLabels != null) {
labels = Nd4j.vstack(ndLabels);
features = Nd4j.vstack(ndFeatures);
} else if (fLabels != null) {
labels = Nd4j.create(fLabels);
features = Nd4j.create(fFeatures);
} else if (dLabels != null) {
labels = Nd4j.create(dLabels);
features = Nd4j.create(dFeatures);
}
DataSet dataSet = new DataSet(features, labels);
try {
queue.add(dataSet);
} catch (Exception e) {
// live with it
}
}
}
}
/**
* Returns the next element in the iteration.
*
* @return the next element in the iteration
* @throws NoSuchElementException if the iteration has no more elements
*/
@Override
public DataSet next() throws NoSuchElementException {
if (queue.isEmpty())
throw new NoSuchElementException();
DataSet dataSet = queue.poll();
if (preProcessor != null)
preProcessor.preProcess(dataSet);
return dataSet;
}
/**
* Removes from the underlying collection the last element returned
* by this iterator (optional operation). This method can be called
* only once per call to {@link #next}. The behavior of an iterator
* is unspecified if the underlying collection is modified while the
* iteration is in progress in any way other than by calling this
* method.
*
* @throws UnsupportedOperationException if the {@code remove}
* operation is not supported by this iterator
* @throws IllegalStateException if the {@code next} method has not
* yet been called, or the {@code remove} method has already
* been called after the last call to the {@code next}
* method
* @implSpec The default implementation throws an instance of
* {@link UnsupportedOperationException} and performs no other action.
*/
@Override
public void remove() {
// no-op
}
@Override
public DataSetPreProcessor getPreProcessor() {
return preProcessor;
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncDataSetIterator.java
+79
View File
@@ -0,0 +1,79 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.extern.slf4j.Slf4j;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.dataset.callbacks.DataSetCallback;
import java.util.concurrent.BlockingQueue;
@Slf4j
@Deprecated
public class AsyncDataSetIterator extends org.nd4j.linalg.dataset.AsyncDataSetIterator {
/**
* Create an Async iterator with the default queue size of 8
* @param baseIterator Underlying iterator to wrap and fetch asynchronously from
*/
public AsyncDataSetIterator(DataSetIterator baseIterator) {
super(baseIterator);
}
/**
* Create an Async iterator with the default queue size of 8
* @param iterator Underlying iterator to wrap and fetch asynchronously from
* @param queue Queue size - number of iterators to
*/
public AsyncDataSetIterator(DataSetIterator iterator, int queueSize, BlockingQueue<DataSet> queue) {
super(iterator, queueSize, queue);
}
public AsyncDataSetIterator(DataSetIterator baseIterator, int queueSize) {
super(baseIterator, queueSize);
}
public AsyncDataSetIterator(DataSetIterator baseIterator, int queueSize, boolean useWorkspace) {
super(baseIterator, queueSize, useWorkspace);
}
public AsyncDataSetIterator(DataSetIterator baseIterator, int queueSize, boolean useWorkspace, Integer deviceId) {
super(baseIterator, queueSize, useWorkspace, deviceId);
}
public AsyncDataSetIterator(DataSetIterator baseIterator, int queueSize, boolean useWorkspace, DataSetCallback callback) {
super(baseIterator, queueSize, useWorkspace, callback);
}
public AsyncDataSetIterator(DataSetIterator iterator, int queueSize, BlockingQueue<DataSet> queue, boolean useWorkspace) {
super(iterator, queueSize, queue, useWorkspace);
}
public AsyncDataSetIterator(DataSetIterator iterator, int queueSize, BlockingQueue<DataSet> queue, boolean useWorkspace, DataSetCallback callback) {
super(iterator, queueSize, queue, useWorkspace, callback);
}
public AsyncDataSetIterator(DataSetIterator iterator, int queueSize, BlockingQueue<DataSet> queue,
boolean useWorkspace, DataSetCallback callback, Integer deviceId) {
super(iterator, queueSize, queue, useWorkspace, callback, deviceId);
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncMultiDataSetIterator.java
+69
View File
@@ -0,0 +1,69 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.extern.slf4j.Slf4j;
import org.nd4j.linalg.dataset.api.MultiDataSet;
import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
import org.nd4j.linalg.dataset.callbacks.DataSetCallback;
import java.util.concurrent.BlockingQueue;
@Slf4j
@Deprecated
public class AsyncMultiDataSetIterator extends org.nd4j.linalg.dataset.AsyncMultiDataSetIterator {
public AsyncMultiDataSetIterator(MultiDataSetIterator baseIterator) {
super(baseIterator);
}
public AsyncMultiDataSetIterator(MultiDataSetIterator iterator, int queueSize, BlockingQueue<MultiDataSet> queue) {
super(iterator, queueSize, queue);
}
public AsyncMultiDataSetIterator(MultiDataSetIterator baseIterator, int queueSize) {
super(baseIterator, queueSize);
}
public AsyncMultiDataSetIterator(MultiDataSetIterator baseIterator, int queueSize, boolean useWorkspace) {
super(baseIterator, queueSize, useWorkspace);
}
public AsyncMultiDataSetIterator(MultiDataSetIterator baseIterator, int queueSize, boolean useWorkspace,
Integer deviceId) {
super(baseIterator, queueSize, useWorkspace, deviceId);
}
public AsyncMultiDataSetIterator(MultiDataSetIterator iterator, int queueSize, BlockingQueue<MultiDataSet> queue,
boolean useWorkspace) {
super(iterator, queueSize, queue, useWorkspace);
}
public AsyncMultiDataSetIterator(MultiDataSetIterator iterator, int queueSize, BlockingQueue<MultiDataSet> queue,
boolean useWorkspace, DataSetCallback callback) {
super(iterator, queueSize, queue, useWorkspace, callback);
}
public AsyncMultiDataSetIterator(MultiDataSetIterator iterator, int queueSize, BlockingQueue<MultiDataSet> queue,
boolean useWorkspace, DataSetCallback callback, Integer deviceId) {
super(iterator, queueSize, queue, useWorkspace, callback, deviceId);
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncShieldDataSetIterator.java
+186
View File
@@ -0,0 +1,186 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.NonNull;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import java.util.List;
public class AsyncShieldDataSetIterator implements DataSetIterator {
private DataSetIterator backingIterator;
/**
* @param iterator Iterator to wrop, to disable asynchronous prefetching for
*/
public AsyncShieldDataSetIterator(@NonNull DataSetIterator iterator) {
this.backingIterator = iterator;
}
/**
* Like the standard next method but allows a
* customizable number of examples returned
*
* @param num the number of examples
* @return the next data applyTransformToDestination
*/
@Override
public DataSet next(int num) {
return backingIterator.next(num);
}
/**
* Input columns for the dataset
*
* @return
*/
@Override
public int inputColumns() {
return backingIterator.inputColumns();
}
/**
* The number of labels for the dataset
*
* @return
*/
@Override
public int totalOutcomes() {
return backingIterator.totalOutcomes();
}
/**
* Is resetting supported by this DataSetIterator? Many DataSetIterators do support resetting,
* but some don't
*
* @return true if reset method is supported; false otherwise
*/
@Override
public boolean resetSupported() {
return backingIterator.resetSupported();
}
/**
* Does this DataSetIterator support asynchronous prefetching of multiple DataSet objects?
*
* PLEASE NOTE: This iterator ALWAYS returns FALSE
*
* @return true if asynchronous prefetching from this iterator is OK; false if asynchronous prefetching should not
* be used with this iterator
*/
@Override
public boolean asyncSupported() {
return false;
}
/**
* Resets the iterator back to the beginning
*/
@Override
public void reset() {
backingIterator.reset();
}
/**
* Batch size
*
* @return
*/
@Override
public int batch() {
return backingIterator.batch();
}
/**
* Set a pre processor
*
* @param preProcessor a pre processor to set
*/
@Override
public void setPreProcessor(DataSetPreProcessor preProcessor) {
backingIterator.setPreProcessor(preProcessor);
}
/**
* Returns preprocessors, if defined
*
* @return
*/
@Override
public DataSetPreProcessor getPreProcessor() {
return backingIterator.getPreProcessor();
}
/**
* Get dataset iterator record reader labels
*/
@Override
public List<String> getLabels() {
return backingIterator.getLabels();
}
/**
* Returns {@code true} if the iteration has more elements.
* (In other words, returns {@code true} if {@link #next} would
* return an element rather than throwing an exception.)
*
* @return {@code true} if the iteration has more elements
*/
@Override
public boolean hasNext() {
return backingIterator.hasNext();
}
/**
* Returns the next element in the iteration.
*
* @return the next element in the iteration
*/
@Override
public DataSet next() {
return backingIterator.next();
}
/**
* Removes from the underlying collection the last element returned
* by this iterator (optional operation). This method can be called
* only once per call to {@link #next}. The behavior of an iterator
* is unspecified if the underlying collection is modified while the
* iteration is in progress in any way other than by calling this
* method.
*
* @throws UnsupportedOperationException if the {@code remove}
* operation is not supported by this iterator
* @throws IllegalStateException if the {@code next} method has not
* yet been called, or the {@code remove} method has already
* been called after the last call to the {@code next}
* method
* @implSpec The default implementation throws an instance of
* {@link UnsupportedOperationException} and performs no other action.
*/
@Override
public void remove() {
// no-op
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncShieldMultiDataSetIterator.java
+136
View File
@@ -0,0 +1,136 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.NonNull;
import org.nd4j.linalg.dataset.api.MultiDataSet;
import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
public class AsyncShieldMultiDataSetIterator implements MultiDataSetIterator {
private MultiDataSetIterator backingIterator;
public AsyncShieldMultiDataSetIterator(@NonNull MultiDataSetIterator iterator) {
this.backingIterator = iterator;
}
/**
* Fetch the next 'num' examples. Similar to the next method, but returns a specified number of examples
*
* @param num Number of examples to fetch
*/
@Override
public MultiDataSet next(int num) {
return backingIterator.next(num);
}
/**
* Set the preprocessor to be applied to each MultiDataSet, before each MultiDataSet is returned.
*
* @param preProcessor MultiDataSetPreProcessor. May be null.
*/
@Override
public void setPreProcessor(MultiDataSetPreProcessor preProcessor) {
backingIterator.setPreProcessor(preProcessor);
}
@Override
public MultiDataSetPreProcessor getPreProcessor() {
return backingIterator.getPreProcessor();
}
/**
* Is resetting supported by this DataSetIterator? Many DataSetIterators do support resetting,
* but some don't
*
* @return true if reset method is supported; false otherwise
*/
@Override
public boolean resetSupported() {
return backingIterator.resetSupported();
}
/**
/**
* Does this DataSetIterator support asynchronous prefetching of multiple DataSet objects?
*
* PLEASE NOTE: This iterator ALWAYS returns FALSE
*
* @return true if asynchronous prefetching from this iterator is OK; false if asynchronous prefetching should not
* be used with this iterator
*/
@Override
public boolean asyncSupported() {
return false;
}
/**
* Resets the iterator back to the beginning
*/
@Override
public void reset() {
backingIterator.reset();
}
/**
* Returns {@code true} if the iteration has more elements.
* (In other words, returns {@code true} if {@link #next} would
* return an element rather than throwing an exception.)
*
* @return {@code true} if the iteration has more elements
*/
@Override
public boolean hasNext() {
return backingIterator.hasNext();
}
/**
* Returns the next element in the iteration.
*
* @return the next element in the iteration
*/
@Override
public MultiDataSet next() {
return backingIterator.next();
}
/**
* Removes from the underlying collection the last element returned
* by this iterator (optional operation). This method can be called
* only once per call to {@link #next}. The behavior of an iterator
* is unspecified if the underlying collection is modified while the
* iteration is in progress in any way other than by calling this
* method.
*
* @throws UnsupportedOperationException if the {@code remove}
* operation is not supported by this iterator
* @throws IllegalStateException if the {@code next} method has not
* yet been called, or the {@code remove} method has already
* been called after the last call to the {@code next}
* method
* @implSpec The default implementation throws an instance of
* {@link UnsupportedOperationException} and performs no other action.
*/
@Override
public void remove() {
// no-op
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/BaseDatasetIterator.java
Executable
+121
View File
@@ -0,0 +1,121 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.Getter;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.dataset.api.iterator.fetcher.BaseDataFetcher;
import java.util.List;
public class BaseDatasetIterator implements DataSetIterator {
protected int batch, numExamples;
protected BaseDataFetcher fetcher;
@Getter
protected DataSetPreProcessor preProcessor;
public BaseDatasetIterator(int batch, int numExamples, BaseDataFetcher fetcher) {
if(batch <= 0){
throw new IllegalArgumentException("Invalid minibatch size: must be > 0 (got: " + batch + ")");
}
this.batch = batch;
if (numExamples < 0)
numExamples = fetcher.totalExamples();
this.numExamples = numExamples;
this.fetcher = fetcher;
}
@Override
public boolean hasNext() {
return fetcher.hasMore() && fetcher.cursor() < numExamples;
}
@Override
public DataSet next() {
fetcher.fetch(batch);
DataSet result = fetcher.next();
if (preProcessor != null) {
preProcessor.preProcess(result);
}
return result;
}
@Override
public DataSet next(int num) {
fetcher.fetch(num);
DataSet next = fetcher.next();
if (preProcessor != null)
preProcessor.preProcess(next);
return next;
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
@Override
public int inputColumns() {
return fetcher.inputColumns();
}
@Override
public int totalOutcomes() {
return fetcher.totalOutcomes();
}
@Override
public boolean resetSupported() {
return true;
}
@Override
public boolean asyncSupported() {
return true;
}
@Override
public void reset() {
fetcher.reset();
}
@Override
public int batch() {
return batch;
}
@Override
public void setPreProcessor(DataSetPreProcessor preProcessor) {
this.preProcessor = preProcessor;
}
@Override
public List<String> getLabels() {
return null;
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/CombinedMultiDataSetPreProcessor.java
+76
View File
@@ -0,0 +1,76 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.NonNull;
import org.nd4j.linalg.dataset.api.MultiDataSet;
import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
import java.util.ArrayList;
import java.util.List;
public class CombinedMultiDataSetPreProcessor implements MultiDataSetPreProcessor {
private List<MultiDataSetPreProcessor> preProcessors;
private CombinedMultiDataSetPreProcessor() {
}
@Override
public void preProcess(MultiDataSet multiDataSet) {
for (MultiDataSetPreProcessor preProcessor : preProcessors) {
preProcessor.preProcess(multiDataSet);
}
}
public static class Builder {
private List<MultiDataSetPreProcessor> preProcessors = new ArrayList<>();
public Builder() {
}
/**
* @param preProcessor to be added to list of preprocessors to be applied
*/
public Builder addPreProcessor(@NonNull MultiDataSetPreProcessor preProcessor) {
preProcessors.add(preProcessor);
return this;
}
/**
* Inserts the specified preprocessor at the specified position to the list of preprocessors to be applied
* @param idx the position to apply the specified preprocessor at
* @param preProcessor to be added to list of preprocessors to be applied
*/
public Builder addPreProcessor(int idx, @NonNull MultiDataSetPreProcessor preProcessor) {
preProcessors.add(idx, preProcessor);
return this;
}
public CombinedMultiDataSetPreProcessor build() {
CombinedMultiDataSetPreProcessor preProcessor = new CombinedMultiDataSetPreProcessor();
preProcessor.preProcessors = this.preProcessors;
return preProcessor;
}
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/CombinedPreProcessor.java
+73
View File
@@ -0,0 +1,73 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.NonNull;
import org.nd4j.linalg.dataset.api.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import java.util.ArrayList;
import java.util.List;
public class CombinedPreProcessor implements DataSetPreProcessor {
private List<DataSetPreProcessor> preProcessors;
private CombinedPreProcessor() {
}
/**
* Pre process a dataset sequentially
*
* @param toPreProcess the data set to pre process
*/
@Override
public void preProcess(DataSet toPreProcess) {
for (DataSetPreProcessor preProcessor : preProcessors) {
preProcessor.preProcess(toPreProcess);
}
}
public static class Builder {
private List<DataSetPreProcessor> preProcessors = new ArrayList<>();
public Builder() {
}
public Builder addPreProcessor(@NonNull DataSetPreProcessor preProcessor) {
preProcessors.add(preProcessor);
return this;
}
public Builder addPreProcessor(int idx, @NonNull DataSetPreProcessor preProcessor) {
preProcessors.add(idx, preProcessor);
return this;
}
public CombinedPreProcessor build() {
CombinedPreProcessor preProcessor = new CombinedPreProcessor();
preProcessor.preProcessors = this.preProcessors;
return preProcessor;
}
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DataSetFetcher.java
Executable
+78
View File
@@ -0,0 +1,78 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import org.nd4j.linalg.dataset.DataSet;
import java.io.Serializable;
@Deprecated
public interface DataSetFetcher extends Serializable {
/**
* Whether the dataset has more to load
* @return whether the data applyTransformToDestination has more to load
*/
boolean hasMore();
/**
* Returns the next data applyTransformToDestination
* @return the next dataset
*/
DataSet next();
/**
* Fetches the next dataset. You need to call this
* to get a new dataset, otherwise {@link #next()}
* just returns the last data applyTransformToDestination fetch
* @param numExamples the number of examples to fetch
*/
void fetch(int numExamples);
/**
* The number of labels for a dataset
* @return the number of labels for a dataset
*/
int totalOutcomes();
/**
* The length of a feature vector for an individual example
* @return the length of a feature vector for an individual example
*/
int inputColumns();
/**
* The total number of examples
* @return the total number of examples
*/
int totalExamples();
/**
* Returns the fetcher back to the beginning of the dataset
*/
void reset();
/**
* Direct access to a number represenative of iterating through a dataset
* @return a cursor similar to an index
*/
int cursor();
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DataSetIteratorSplitter.java
+344
View File
@@ -0,0 +1,344 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.exception.ND4JIllegalStateException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@Slf4j
public class DataSetIteratorSplitter {
protected DataSetIterator backedIterator;
protected final long totalExamples;
protected final double ratio;
protected final double[] ratios;
protected final long numTrain;
protected final long numTest;
protected final long numArbitrarySets;
protected final int[] splits;
protected AtomicLong counter = new AtomicLong(0);
protected AtomicBoolean resetPending = new AtomicBoolean(false);
protected DataSet firstTrain = null;
/**
* The only constructor
*
* @param baseIterator - iterator to be wrapped and split
* @param totalBatches - total batches in baseIterator
* @param ratio - train/test split ratio
*/
public DataSetIteratorSplitter(@NonNull DataSetIterator baseIterator, long totalBatches, double ratio) {
if (!(ratio > 0.0 && ratio < 1.0))
throw new ND4JIllegalStateException("Ratio value should be in range of 0.0 > X < 1.0");
if (totalBatches < 0)
throw new ND4JIllegalStateException("totalExamples number should be positive value");
if (!baseIterator.resetSupported())
throw new ND4JIllegalStateException("Underlying iterator doesn't support reset, so it can't be used for runtime-split");
this.backedIterator = baseIterator;
this.totalExamples = totalBatches;
this.ratio = ratio;
ratios = new double[]{ratio, 1 - ratio};
this.numTrain = (long) (totalExamples * ratio);
this.numTest = totalExamples - numTrain;
this.numArbitrarySets = 2;
this.splits = new int[this.ratios.length];
for (int i = 0; i < this.splits.length; ++i) {
this.splits[i] = (int)(totalExamples * ratios[i]);
}
log.warn("IteratorSplitter is used: please ensure you don't use randomization/shuffle in underlying iterator!");
}
public DataSetIteratorSplitter(@NonNull DataSetIterator baseIterator, long totalBatches, double[] ratios) {
for (double ratio : ratios) {
if (!(ratio > 0.0 && ratio < 1.0))
throw new ND4JIllegalStateException("Ratio value should be in range of 0.0 > X < 1.0");
}
if (totalBatches < 0)
throw new ND4JIllegalStateException("totalExamples number should be positive value");
if (!baseIterator.resetSupported())
throw new ND4JIllegalStateException("Underlying iterator doesn't support reset, so it can't be used for runtime-split");
this.backedIterator = baseIterator;
this.totalExamples = totalBatches;
this.ratio = 0.0;
this.ratios = ratios;
this.numTrain = 0; //(long) (totalExamples * ratio);
this.numTest = 0; //totalExamples - numTrain;
this.numArbitrarySets = ratios.length;
this.splits = new int[this.ratios.length];
for (int i = 0; i < this.splits.length; ++i) {
this.splits[i] = (int)(totalExamples * ratios[i]);
}
log.warn("IteratorSplitter is used: please ensure you don't use randomization/shuffle in underlying iterator!");
}
public DataSetIteratorSplitter(@NonNull DataSetIterator baseIterator, int[] splits) {
int totalBatches = 0;
for (val v:splits)
totalBatches += v;
if (totalBatches < 0)
throw new ND4JIllegalStateException("totalExamples number should be positive value");
if (!baseIterator.resetSupported())
throw new ND4JIllegalStateException("Underlying iterator doesn't support reset, so it can't be used for runtime-split");
this.backedIterator = baseIterator;
this.totalExamples = totalBatches;
this.ratio = 0.0;
this.ratios = null;
this.numTrain = 0;
this.numTest = 0;
this.splits = splits;
this.numArbitrarySets = splits.length;
log.warn("IteratorSplitter is used: please ensure you don't use randomization/shuffle in underlying iterator!");
}
public List<DataSetIterator> getIterators() {
List<DataSetIterator> retVal = new ArrayList<>();
int partN = 0;
int bottom = 0;
for (final int split : splits) {
ScrollableDataSetIterator partIterator =
new ScrollableDataSetIterator(partN++,
backedIterator,
counter,
resetPending,
firstTrain,
new int[]{bottom,split});
bottom += split;
retVal.add(partIterator);
}
return retVal;
}
/**
* This method returns train iterator instance
*
* @return
*/
@Deprecated
public DataSetIterator getTrainIterator() {
return new DataSetIterator() {
@Override
public DataSet next(int i) {
throw new UnsupportedOperationException();
}
@Override
public List<String> getLabels() {
return backedIterator.getLabels();
}
@Override
public int inputColumns() {
return backedIterator.inputColumns();
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
@Override
public int totalOutcomes() {
return backedIterator.totalOutcomes();
}
@Override
public boolean resetSupported() {
return backedIterator.resetSupported();
}
@Override
public boolean asyncSupported() {
return backedIterator.asyncSupported();
}
@Override
public void reset() {
resetPending.set(true);
}
@Override
public int batch() {
return backedIterator.batch();
}
@Override
public void setPreProcessor(DataSetPreProcessor dataSetPreProcessor) {
backedIterator.setPreProcessor(dataSetPreProcessor);
}
@Override
public DataSetPreProcessor getPreProcessor() {
return backedIterator.getPreProcessor();
}
@Override
public boolean hasNext() {
if (resetPending.get()) {
if (resetSupported()) {
backedIterator.reset();
counter.set(0);
resetPending.set(false);
} else
throw new UnsupportedOperationException("Reset isn't supported by underlying iterator");
}
val state = backedIterator.hasNext();
if (state && counter.get() < numTrain)
return true;
else
return false;
}
@Override
public DataSet next() {
counter.incrementAndGet();
val p = backedIterator.next();
if (counter.get() == 1 && firstTrain == null) {
// first epoch ever, we'll save first dataset and will use it to check for equality later
firstTrain = p.copy();
firstTrain.detach();
} else if (counter.get() == 1) {
// epoch > 1, comparing first dataset to previously stored dataset. they should be equal
int cnt = 0;
if (!p.getFeatures().equalsWithEps(firstTrain.getFeatures(), 1e-5))
throw new ND4JIllegalStateException("First examples do not match. Randomization was used?");
}
return p;
}
};
}
/**
* This method returns test iterator instance
*
* @return
*/
@Deprecated
public DataSetIterator getTestIterator() {
return new DataSetIterator() {
@Override
public DataSet next(int i) {
throw new UnsupportedOperationException();
}
@Override
public List<String> getLabels() {
return backedIterator.getLabels();
}
@Override
public int inputColumns() {
return backedIterator.inputColumns();
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
@Override
public int totalOutcomes() {
return backedIterator.totalOutcomes();
}
@Override
public boolean resetSupported() {
return backedIterator.resetSupported();
}
@Override
public boolean asyncSupported() {
return backedIterator.asyncSupported();
}
@Override
public void reset() {
resetPending.set(true);
}
@Override
public int batch() {
return backedIterator.batch();
}
@Override
public void setPreProcessor(DataSetPreProcessor dataSetPreProcessor) {
backedIterator.setPreProcessor(dataSetPreProcessor);
}
@Override
public DataSetPreProcessor getPreProcessor() {
return backedIterator.getPreProcessor();
}
@Override
public boolean hasNext() {
val state = backedIterator.hasNext();
if (state && counter.get() < numTrain + numTest)
return true;
else
return false;
}
@Override
public DataSet next() {
counter.incrementAndGet();
return backedIterator.next();
}
};
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DoublesDataSetIterator.java
+35
View File
@@ -0,0 +1,35 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.NonNull;
import org.nd4j.common.primitives.Pair;
public class DoublesDataSetIterator extends AbstractDataSetIterator<double[]> {
/**
* @param iterable Iterable to source data from
* @param batchSize Batch size for generated DataSet objects
*/
public DoublesDataSetIterator(@NonNull Iterable<Pair<double[], double[]>> iterable, int batchSize) {
super(iterable, batchSize);
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyBlockDataSetIterator.java
+56
View File
@@ -0,0 +1,56 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.nd4j.linalg.dataset.api.DataSet;
import org.nd4j.linalg.dataset.api.iterator.BlockDataSetIterator;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import java.util.ArrayList;
@Slf4j
public class DummyBlockDataSetIterator implements BlockDataSetIterator {
protected final DataSetIterator iterator;
public DummyBlockDataSetIterator(@NonNull DataSetIterator iterator) {
this.iterator = iterator;
}
@Override
public boolean hasAnything() {
return iterator.hasNext();
}
@Override
public DataSet[] next(int maxDatasets) {
val list = new ArrayList<DataSet>(maxDatasets);
int cnt = 0;
while (iterator.hasNext() && cnt < maxDatasets) {
list.add(iterator.next());
cnt++;
}
return list.toArray(new DataSet[list.size()]);
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyBlockMultiDataSetIterator.java
+59
View File
@@ -0,0 +1,59 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.nd4j.linalg.dataset.api.DataSet;
import org.nd4j.linalg.dataset.api.MultiDataSet;
import org.nd4j.linalg.dataset.api.iterator.BlockDataSetIterator;
import org.nd4j.linalg.dataset.api.iterator.BlockMultiDataSetIterator;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
import java.util.ArrayList;
@Slf4j
public class DummyBlockMultiDataSetIterator implements BlockMultiDataSetIterator {
protected final MultiDataSetIterator iterator;
public DummyBlockMultiDataSetIterator(@NonNull MultiDataSetIterator iterator) {
this.iterator = iterator;
}
@Override
public boolean hasAnything() {
return iterator.hasNext();
}
@Override
public MultiDataSet[] next(int maxDatasets) {
val list = new ArrayList<MultiDataSet>(maxDatasets);
int cnt = 0;
while (iterator.hasNext() && cnt < maxDatasets) {
list.add(iterator.next());
cnt++;
}
return list.toArray(new MultiDataSet[list.size()]);
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyPreProcessor.java
+36
View File
@@ -0,0 +1,36 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import org.nd4j.linalg.dataset.api.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
public class DummyPreProcessor implements DataSetPreProcessor {
/**
* Pre process a dataset
*
* @param toPreProcess the data set to pre process
*/
@Override
public void preProcess(DataSet toPreProcess) {
// no-op
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/EarlyTerminationDataSetIterator.java
+124
View File
@@ -0,0 +1,124 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import java.util.List;
public class EarlyTerminationDataSetIterator implements DataSetIterator {
private DataSetIterator underlyingIterator;
private int terminationPoint;
private int minibatchCount = 0;
/**
* Constructor takes the iterator to wrap and the number of minibatches after which the call to hasNext()
* will return false
* @param underlyingIterator, iterator to wrap
* @param terminationPoint, minibatches after which hasNext() will return false
*/
public EarlyTerminationDataSetIterator(DataSetIterator underlyingIterator, int terminationPoint) {
if (terminationPoint <= 0)
throw new IllegalArgumentException(
"Termination point (the number of calls to .next() or .next(num)) has to be > 0");
this.underlyingIterator = underlyingIterator;
this.terminationPoint = terminationPoint;
}
@Override
public DataSet next(int num) {
if (minibatchCount < terminationPoint) {
minibatchCount++;
return underlyingIterator.next(num);
} else {
throw new RuntimeException("Calls to next have exceeded termination point.");
}
}
@Override
public int inputColumns() {
return underlyingIterator.inputColumns();
}
@Override
public int totalOutcomes() {
return underlyingIterator.totalOutcomes();
}
@Override
public boolean resetSupported() {
return underlyingIterator.resetSupported();
}
@Override
public boolean asyncSupported() {
return underlyingIterator.asyncSupported();
}
@Override
public void reset() {
minibatchCount = 0;
underlyingIterator.reset();
}
@Override
public int batch() {
return underlyingIterator.batch();
}
@Override
public void setPreProcessor(DataSetPreProcessor preProcessor) {
underlyingIterator.setPreProcessor(preProcessor);
}
@Override
public DataSetPreProcessor getPreProcessor() {
return underlyingIterator.getPreProcessor();
}
@Override
public List<String> getLabels() {
return underlyingIterator.getLabels();
}
@Override
public boolean hasNext() {
return underlyingIterator.hasNext() && minibatchCount < terminationPoint;
}
@Override
public DataSet next() {
if (minibatchCount < terminationPoint) {
minibatchCount++;
return underlyingIterator.next();
} else {
throw new RuntimeException("Calls to next have exceeded the allotted number of minibatches.");
}
}
@Override
public void remove() {
underlyingIterator.remove();
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/EarlyTerminationMultiDataSetIterator.java
+102
View File
@@ -0,0 +1,102 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import org.nd4j.linalg.dataset.api.MultiDataSet;
import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
public class EarlyTerminationMultiDataSetIterator implements MultiDataSetIterator {
private MultiDataSetIterator underlyingIterator;
private int terminationPoint;
private int minibatchCount = 0;
/**
* Constructor takes the iterator to wrap and the number of minibatches after which the call to hasNext()
* will return false
* @param underlyingIterator, iterator to wrap
* @param terminationPoint, minibatches after which hasNext() will return false
*/
public EarlyTerminationMultiDataSetIterator(MultiDataSetIterator underlyingIterator, int terminationPoint) {
if (terminationPoint <= 0)
throw new IllegalArgumentException(
"Termination point (the number of calls to .next() or .next(num)) has to be > 0");
this.underlyingIterator = underlyingIterator;
this.terminationPoint = terminationPoint;
}
@Override
public MultiDataSet next(int num) {
if (minibatchCount < terminationPoint) {
minibatchCount++;
return underlyingIterator.next(num);
} else {
throw new RuntimeException("Calls to next have exceeded termination point.");
}
}
@Override
public void setPreProcessor(MultiDataSetPreProcessor preProcessor) {
underlyingIterator.setPreProcessor(preProcessor);
}
@Override
public MultiDataSetPreProcessor getPreProcessor() {
return underlyingIterator.getPreProcessor();
}
@Override
public boolean resetSupported() {
return underlyingIterator.resetSupported();
}
@Override
public boolean asyncSupported() {
return underlyingIterator.asyncSupported();
}
@Override
public void reset() {
minibatchCount = 0;
underlyingIterator.reset();
}
@Override
public boolean hasNext() {
return underlyingIterator.hasNext() && minibatchCount < terminationPoint;
}
@Override
public MultiDataSet next() {
if (minibatchCount < terminationPoint) {
minibatchCount++;
return underlyingIterator.next();
} else {
throw new RuntimeException("Calls to next have exceeded the allotted number of minibatches.");
}
}
@Override
public void remove() {
underlyingIterator.remove();
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ExistingDataSetIterator.java
+170
View File
@@ -0,0 +1,170 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.Getter;
import lombok.NonNull;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import java.util.Iterator;
import java.util.List;
public class ExistingDataSetIterator implements DataSetIterator {
@Getter
private DataSetPreProcessor preProcessor;
private transient Iterable<DataSet> iterable;
private transient Iterator<DataSet> iterator;
private int totalExamples = 0;
private int numFeatures = 0;
private int numLabels = 0;
private List<String> labels;
/**
* Note that when using this constructor, resetting is not supported
* @param iterator Iterator to wrap
*/
public ExistingDataSetIterator(@NonNull Iterator<DataSet> iterator) {
this.iterator = iterator;
}
/**
* Note that when using this constructor, resetting is not supported
* @param iterator Iterator to wrap
* @param labels String labels. May be null.
*/
public ExistingDataSetIterator(@NonNull Iterator<DataSet> iterator, @NonNull List<String> labels) {
this(iterator);
this.labels = labels;
}
/**
* Wraps the specified iterable. Supports resetting
* @param iterable Iterable to wrap
*/
public ExistingDataSetIterator(@NonNull Iterable<DataSet> iterable) {
this.iterable = iterable;
this.iterator = iterable.iterator();
}
/**
* Wraps the specified iterable. Supports resetting
* @param iterable Iterable to wrap
* @param labels Labels list. May be null
*/
public ExistingDataSetIterator(@NonNull Iterable<DataSet> iterable, @NonNull List<String> labels) {
this(iterable);
this.labels = labels;
}
public ExistingDataSetIterator(@NonNull Iterable<DataSet> iterable, int totalExamples, int numFeatures,
int numLabels) {
this(iterable);
this.totalExamples = totalExamples;
this.numFeatures = numFeatures;
this.numLabels = numLabels;
}
@Override
public DataSet next(int num) {
// TODO: this might be changed
throw new UnsupportedOperationException("next(int) isn't supported");
}
@Override
public int inputColumns() {
return numFeatures;
}
@Override
public int totalOutcomes() {
if (labels != null)
return labels.size();
return numLabels;
}
@Override
public boolean resetSupported() {
return iterable != null;
}
@Override
public boolean asyncSupported() {
//No need to asynchronously prefetch here: already in memory
return false;
}
@Override
public void reset() {
if (iterable != null)
this.iterator = iterable.iterator();
else
throw new IllegalStateException(
"To use reset() method you need to provide Iterable<DataSet>, not Iterator");
}
@Override
public int batch() {
return 0;
}
@Override
public void setPreProcessor(DataSetPreProcessor preProcessor) {
this.preProcessor = preProcessor;
}
@Override
public List<String> getLabels() {
return labels;
}
@Override
public boolean hasNext() {
if (iterator != null)
return iterator.hasNext();
return false;
}
@Override
public DataSet next() {
if (preProcessor != null) {
DataSet ds = iterator.next();
if (!ds.isPreProcessed()) {
preProcessor.preProcess(ds);
ds.markAsPreProcessed();
}
return ds;
} else
return iterator.next();
}
@Override
public void remove() {
// no-op
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/FileSplitDataSetIterator.java
+129
View File
@@ -0,0 +1,129 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.deeplearning4j.datasets.iterator.callbacks.FileCallback;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import java.io.File;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
@Slf4j
public class FileSplitDataSetIterator implements DataSetIterator {
private DataSetPreProcessor preProcessor;
private List<File> files;
private int numFiles;
private AtomicInteger counter = new AtomicInteger(0);
private FileCallback callback;
/**
* @param files List of files to iterate over
* @param callback Callback for loading the files
*/
public FileSplitDataSetIterator(@NonNull List<File> files, @NonNull FileCallback callback) {
this.files = files;
this.numFiles = files.size();
this.callback = callback;
}
@Override
public DataSet next(int num) {
throw new UnsupportedOperationException();
}
@Override
public int inputColumns() {
return 0;
}
@Override
public int totalOutcomes() {
return 0;
}
@Override
public boolean resetSupported() {
return true;
}
@Override
public boolean asyncSupported() {
return true;
}
@Override
public void reset() {
counter.set(0);
}
@Override
public int batch() {
return 0;
}
@Override
public void setPreProcessor(DataSetPreProcessor preProcessor) {
this.preProcessor = preProcessor;
}
@Override
public DataSetPreProcessor getPreProcessor() {
return preProcessor;
}
@Override
public List<String> getLabels() {
return null;
}
@Override
public boolean hasNext() {
return counter.get() < numFiles;
}
@Override
public DataSet next() {
// long time1 = System.nanoTime();
DataSet ds = callback.call(files.get(counter.getAndIncrement()));
if (preProcessor != null && ds != null)
preProcessor.preProcess(ds);
// long time2 = System.nanoTime();
// if (counter.get() % 5 == 0)
// log.info("Device: [{}]; Time: [{}] ns;", Nd4j.getAffinityManager().getDeviceForCurrentThread(), time2 - time1);
return ds;
}
@Override
public void remove() {
// no-op
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/FloatsDataSetIterator.java
+35
View File
@@ -0,0 +1,35 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.NonNull;
import org.nd4j.common.primitives.Pair;
public class FloatsDataSetIterator extends AbstractDataSetIterator<float[]> {
/**
* @param iterable Iterable to source data from
* @param batchSize Batch size for generated DataSet objects
*/
public FloatsDataSetIterator(@NonNull Iterable<Pair<float[], float[]>> iterable, int batchSize) {
super(iterable, batchSize);
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/INDArrayDataSetIterator.java
+36
View File
@@ -0,0 +1,36 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.NonNull;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.common.primitives.Pair;
public class INDArrayDataSetIterator extends AbstractDataSetIterator<INDArray> {
/**
* @param iterable Iterable to source data from
* @param batchSize Batch size for generated DataSet objects
*/
public INDArrayDataSetIterator(@NonNull Iterable<Pair<INDArray, INDArray>> iterable, int batchSize) {
super(iterable, batchSize);
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/IteratorDataSetIterator.java
+173
View File
@@ -0,0 +1,173 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.Getter;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import java.util.*;
public class IteratorDataSetIterator implements DataSetIterator {
private final Iterator<DataSet> iterator;
private final int batchSize;
private final LinkedList<DataSet> queued; //Used when splitting larger examples than we want to return in a batch
@Getter
private DataSetPreProcessor preProcessor;
private int inputColumns = -1;
private int totalOutcomes = -1;
private int cursor = 0;
public IteratorDataSetIterator(Iterator<DataSet> iterator, int batchSize) {
this.iterator = iterator;
this.batchSize = batchSize;
this.queued = new LinkedList<>();
}
@Override
public boolean hasNext() {
return !queued.isEmpty() || iterator.hasNext();
}
@Override
public DataSet next() {
return next(batchSize);
}
@Override
public DataSet next(int num) {
if (!hasNext())
throw new NoSuchElementException();
List<DataSet> list = new ArrayList<>();
int countSoFar = 0;
while ((!queued.isEmpty() || iterator.hasNext()) && countSoFar < batchSize) {
DataSet next;
if (!queued.isEmpty()) {
next = queued.removeFirst();
} else {
next = iterator.next();
}
int nExamples = next.numExamples();
if (countSoFar + nExamples <= batchSize) {
//Add the entire DataSet as-is
list.add(next);
} else {
//Otherwise, split it
DataSet toKeep = (DataSet) next.getRange(0, batchSize - countSoFar);
DataSet toCache = (DataSet) next.getRange(batchSize - countSoFar, nExamples);
list.add(toKeep);
queued.add(toCache);
}
countSoFar += nExamples;
}
if (inputColumns == -1) {
//Set columns etc for later use
DataSet temp = list.get(0);
inputColumns = (int) temp.getFeatures().size(1);
totalOutcomes = temp.getLabels() == null ? 0 : (int) temp.getLabels().size(1); //May be null for layerwise pretraining
}
DataSet out;
if (list.size() == 1) {
out = list.get(0);
} else {
out = DataSet.merge(list);
}
if (preProcessor != null) {
if (!out.isPreProcessed()) {
preProcessor.preProcess(out);
out.markAsPreProcessed();
}
}
cursor += out.numExamples();
return out;
}
@Override
public int inputColumns() {
if (inputColumns != -1)
return inputColumns;
prefetchBatchSetInputOutputValues();
return inputColumns;
}
@Override
public int totalOutcomes() {
if (totalOutcomes != -1)
return totalOutcomes;
prefetchBatchSetInputOutputValues();
return totalOutcomes;
}
@Override
public boolean resetSupported() {
return false;
}
@Override
public boolean asyncSupported() {
return true;
}
@Override
public void reset() {
throw new UnsupportedOperationException("Reset not supported");
}
@Override
public int batch() {
return batchSize;
}
@Override
public void setPreProcessor(DataSetPreProcessor preProcessor) {
this.preProcessor = preProcessor;
}
@Override
public List<String> getLabels() {
return null;
}
@Override
public void remove() {
throw new UnsupportedOperationException("Not supported");
}
private void prefetchBatchSetInputOutputValues() {
if (!iterator.hasNext())
return;
DataSet next = iterator.next();
inputColumns = (int) next.getFeatures().size(1);
totalOutcomes = (int) next.getLabels().size(1);
queued.add(next);
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/IteratorMultiDataSetIterator.java
+185
View File
@@ -0,0 +1,185 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.api.MultiDataSet;
import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
import org.nd4j.linalg.indexing.NDArrayIndex;
import java.util.*;
public class IteratorMultiDataSetIterator implements MultiDataSetIterator {
private final Iterator<MultiDataSet> iterator;
private final int batchSize;
private final LinkedList<MultiDataSet> queued; //Used when splitting larger examples than we want to return in a batch
private MultiDataSetPreProcessor preProcessor;
public IteratorMultiDataSetIterator(Iterator<MultiDataSet> iterator, int batchSize) {
this.iterator = iterator;
this.batchSize = batchSize;
this.queued = new LinkedList<>();
}
@Override
public boolean hasNext() {
return !queued.isEmpty() || iterator.hasNext();
}
@Override
public MultiDataSet next() {
return next(batchSize);
}
@Override
public MultiDataSet next(int num) {
if (!hasNext())
throw new NoSuchElementException();
List<MultiDataSet> list = new ArrayList<>();
int countSoFar = 0;
while ((!queued.isEmpty() || iterator.hasNext()) && countSoFar < batchSize) {
MultiDataSet next;
if (!queued.isEmpty()) {
next = queued.removeFirst();
} else {
next = iterator.next();
}
long nExamples = next.getFeatures(0).size(0);
if (countSoFar + nExamples <= batchSize) {
//Add the entire MultiDataSet as-is
list.add(next);
} else {
//Split the MultiDataSet
int nFeatures = next.numFeatureArrays();
int nLabels = next.numLabelsArrays();
INDArray[] fToKeep = new INDArray[nFeatures];
INDArray[] lToKeep = new INDArray[nLabels];
INDArray[] fToCache = new INDArray[nFeatures];
INDArray[] lToCache = new INDArray[nLabels];
INDArray[] fMaskToKeep = (next.getFeaturesMaskArrays() != null ? new INDArray[nFeatures] : null);
INDArray[] lMaskToKeep = (next.getLabelsMaskArrays() != null ? new INDArray[nLabels] : null);
INDArray[] fMaskToCache = (next.getFeaturesMaskArrays() != null ? new INDArray[nFeatures] : null);
INDArray[] lMaskToCache = (next.getLabelsMaskArrays() != null ? new INDArray[nLabels] : null);
for (int i = 0; i < nFeatures; i++) {
INDArray fi = next.getFeatures(i);
fToKeep[i] = getRange(fi, 0, batchSize - countSoFar);
fToCache[i] = getRange(fi, batchSize - countSoFar, nExamples);
if (fMaskToKeep != null) {
INDArray fmi = next.getFeaturesMaskArray(i);
fMaskToKeep[i] = getRange(fmi, 0, batchSize - countSoFar);
fMaskToCache[i] = getRange(fmi, batchSize - countSoFar, nExamples);
}
}
for (int i = 0; i < nLabels; i++) {
INDArray li = next.getLabels(i);
lToKeep[i] = getRange(li, 0, batchSize - countSoFar);
lToCache[i] = getRange(li, batchSize - countSoFar, nExamples);
if (lMaskToKeep != null) {
INDArray lmi = next.getLabelsMaskArray(i);
lMaskToKeep[i] = getRange(lmi, 0, batchSize - countSoFar);
lMaskToCache[i] = getRange(lmi, batchSize - countSoFar, nExamples);
}
}
MultiDataSet toKeep =
new org.nd4j.linalg.dataset.MultiDataSet(fToKeep, lToKeep, fMaskToKeep, lMaskToKeep);
MultiDataSet toCache = new org.nd4j.linalg.dataset.MultiDataSet(fToCache, lToCache, fMaskToCache,
lMaskToCache);
list.add(toKeep);
queued.add(toCache);
}
countSoFar += nExamples;
}
MultiDataSet out;
if (list.size() == 1) {
out = list.get(0);
} else {
out = org.nd4j.linalg.dataset.MultiDataSet.merge(list);
}
if (preProcessor != null)
preProcessor.preProcess(out);
return out;
}
private static INDArray getRange(INDArray arr, long exampleFrom, long exampleToExclusive) {
if (arr == null)
return null;
int rank = arr.rank();
switch (rank) {
case 2:
return arr.get(NDArrayIndex.interval(exampleFrom, exampleToExclusive), NDArrayIndex.all());
case 3:
return arr.get(NDArrayIndex.interval(exampleFrom, exampleToExclusive), NDArrayIndex.all(),
NDArrayIndex.all());
case 4:
return arr.get(NDArrayIndex.interval(exampleFrom, exampleToExclusive), NDArrayIndex.all(),
NDArrayIndex.all(), NDArrayIndex.all());
default:
throw new RuntimeException("Invalid rank: " + rank);
}
}
@Override
public boolean resetSupported() {
return false;
}
@Override
public boolean asyncSupported() {
//No need to asynchronously prefetch here: already in memory
return false;
}
@Override
public void reset() {
throw new UnsupportedOperationException("Reset not supported");
}
@Override
public void setPreProcessor(MultiDataSetPreProcessor preProcessor) {
this.preProcessor = preProcessor;
}
@Override
public MultiDataSetPreProcessor getPreProcessor() {
return preProcessor;
}
@Override
public void remove() {
throw new UnsupportedOperationException("Not supported");
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/JointMultiDataSetIterator.java
+233
View File
@@ -0,0 +1,233 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.AllArgsConstructor;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.api.MultiDataSet;
import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@Slf4j
@NoArgsConstructor
@AllArgsConstructor
public class JointMultiDataSetIterator implements MultiDataSetIterator {
protected MultiDataSetPreProcessor preProcessor;
protected Collection<DataSetIterator> iterators;
protected int outcome = -1;
/**
* @param iterators Underlying iterators to wrap
*/
public JointMultiDataSetIterator(DataSetIterator... iterators) {
this.iterators = new ArrayList<DataSetIterator>();
this.iterators.addAll(Arrays.asList(iterators));
this.outcome = -1;
}
/**
*
* @param outcome Index to get the label from. If < 0, labels from all iterators will be used to create the
* final MultiDataSet
* @param iterators Underlying iterators to wrap
*/
public JointMultiDataSetIterator(int outcome, DataSetIterator... iterators){
this(iterators);
this.outcome = outcome;
}
/**
* Fetch the next 'num' examples. Similar to the next method, but returns a specified number of examples
*
* @param num Number of examples to fetch
*/
@Override
public MultiDataSet next(int num) {
throw new UnsupportedOperationException();
}
/**
* Set the preprocessor to be applied to each MultiDataSet, before each MultiDataSet is returned.
*
* @param preProcessor MultiDataSetPreProcessor. May be null.
*/
@Override
public void setPreProcessor(MultiDataSetPreProcessor preProcessor) {
this.preProcessor = preProcessor;
}
/**
* Get the {@link MultiDataSetPreProcessor}, if one has previously been set.
* Returns null if no preprocessor has been set
*
* @return Preprocessor
*/
@Override
public MultiDataSetPreProcessor getPreProcessor() {
return preProcessor;
}
/**
* Is resetting supported by this DataSetIterator? Many DataSetIterators do support resetting,
* but some don't
*
* @return true if reset method is supported; false otherwise
*/
@Override
public boolean resetSupported() {
boolean sup = true;
for (val i: iterators)
if (!i.resetSupported()) {
sup = false;
break;
}
return sup;
}
/**
* Does this MultiDataSetIterator support asynchronous prefetching of multiple MultiDataSet objects?
* Most MultiDataSetIterators do, but in some cases it may not make sense to wrap this iterator in an
* iterator that does asynchronous prefetching. For example, it would not make sense to use asynchronous
* prefetching for the following types of iterators:
* (a) Iterators that store their full contents in memory already
* (b) Iterators that re-use features/labels arrays (as future next() calls will overwrite past contents)
* (c) Iterators that already implement some level of asynchronous prefetching
* (d) Iterators that may return different data depending on when the next() method is called
*
* @return true if asynchronous prefetching from this iterator is OK; false if asynchronous prefetching should not
* be used with this iterator
*/
@Override
public boolean asyncSupported() {
boolean sup = true;
for (val i: iterators)
if (!i.asyncSupported()) {
sup = false;
break;
}
return sup;
}
/**
* Resets the iterator back to the beginning
*/
@Override
public void reset() {
for (val i: iterators)
i.reset();
}
/**
* Returns {@code true} if the iteration has more elements.
* (In other words, returns {@code true} if {@link #next} would
* return an element rather than throwing an exception.)
*
* @return {@code true} if the iteration has more elements
*/
@Override
public boolean hasNext() {
boolean has = true;
for (val i: iterators)
if (!i.hasNext()) {
has = false;
break;
}
return has;
}
/**
* Returns the next element in the iteration.
*
* @return the next element in the iteration
*/
@Override
public MultiDataSet next() {
val features = new ArrayList<INDArray>();
val labels = new ArrayList<INDArray>();
val featuresMask = new ArrayList<INDArray>();
val labelsMask = new ArrayList<INDArray>();
boolean hasFM = false;
boolean hasLM = false;
int cnt = 0;
for (val i: iterators) {
val ds = i.next();
features.add(ds.getFeatures());
featuresMask.add(ds.getFeaturesMaskArray());
if (outcome < 0 || cnt == outcome) {
labels.add(ds.getLabels());
labelsMask.add(ds.getLabelsMaskArray());
}
if (ds.getFeaturesMaskArray() != null)
hasFM = true;
if (ds.getLabelsMaskArray() != null)
hasLM = true;
cnt++;
}
INDArray[] fm = hasFM ? featuresMask.toArray(new INDArray[0]) : null;
INDArray[] lm = hasLM ? labelsMask.toArray(new INDArray[0]) : null;
val mds = new org.nd4j.linalg.dataset.MultiDataSet(features.toArray(new INDArray[0]), labels.toArray(new INDArray[0]), fm, lm);
if (preProcessor != null)
preProcessor.preProcess(mds);
return mds;
}
/**
* PLEASE NOTE: This method is NOT implemented
*
* @throws UnsupportedOperationException if the {@code remove}
* operation is not supported by this iterator
* @throws IllegalStateException if the {@code next} method has not
* yet been called, or the {@code remove} method has already
* been called after the last call to the {@code next}
* method
* @implSpec The default implementation throws an instance of
* {@link UnsupportedOperationException} and performs no other action.
*/
@Override
public void remove() {
// noopp
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultiDataSetIteratorSplitter.java
+292
View File
@@ -0,0 +1,292 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.nd4j.linalg.dataset.api.MultiDataSet;
import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
import org.nd4j.linalg.exception.ND4JIllegalStateException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@Slf4j
public class MultiDataSetIteratorSplitter {
protected MultiDataSetIterator backedIterator;
protected final long totalExamples;
protected final double ratio;
protected final long numTrain;
protected final long numTest;
protected final double[] ratios;
protected final long numArbitrarySets;
protected final int[] splits;
protected AtomicLong counter = new AtomicLong(0);
protected AtomicBoolean resetPending = new AtomicBoolean(false);
protected org.nd4j.linalg.dataset.MultiDataSet firstTrain = null;
/**
*
* @param baseIterator
* @param totalBatches - total number of batches in underlying iterator. this value will be used to determine number of test/train batches
* @param ratio - this value will be used as splitter. should be between in range of 0.0 > X < 1.0. I.e. if value 0.7 is provided, then 70% of total examples will be used for training, and 30% of total examples will be used for testing
*/
public MultiDataSetIteratorSplitter(@NonNull MultiDataSetIterator baseIterator, long totalBatches, double ratio) {
if (!(ratio > 0.0 && ratio < 1.0))
throw new ND4JIllegalStateException("Ratio value should be in range of 0.0 > X < 1.0");
if (totalBatches < 0)
throw new ND4JIllegalStateException("totalExamples number should be positive value");
if (!baseIterator.resetSupported())
throw new ND4JIllegalStateException("Underlying iterator doesn't support reset, so it can't be used for runtime-split");
this.backedIterator = baseIterator;
this.totalExamples = totalBatches;
this.ratio = ratio;
this.numTrain = (long) (totalExamples * ratio);
this.numTest = totalExamples - numTrain;
this.ratios = null;
this.numArbitrarySets = 0;
this.splits = null;
log.warn("IteratorSplitter is used: please ensure you don't use randomization/shuffle in underlying iterator!");
}
public MultiDataSetIteratorSplitter(@NonNull MultiDataSetIterator baseIterator, long totalBatches, double[] ratios) {
for (double ratio : ratios) {
if (!(ratio > 0.0 && ratio < 1.0))
throw new ND4JIllegalStateException("Ratio value should be in range of 0.0 > X < 1.0");
}
if (totalBatches < 0)
throw new ND4JIllegalStateException("totalExamples number should be positive value");
if (!baseIterator.resetSupported())
throw new ND4JIllegalStateException("Underlying iterator doesn't support reset, so it can't be used for runtime-split");
this.backedIterator = baseIterator;
this.totalExamples = totalBatches;
this.ratio = 0.0;
this.numTrain = (long) (totalExamples * ratio);
this.numTest = totalExamples - numTrain;
this.ratios = null;
this.numArbitrarySets = ratios.length;
this.splits = new int[this.ratios.length];
for (int i = 0; i < this.splits.length; ++i) {
this.splits[i] = (int)(totalExamples * ratios[i]);
}
log.warn("IteratorSplitter is used: please ensure you don't use randomization/shuffle in underlying iterator!");
}
public MultiDataSetIteratorSplitter(@NonNull MultiDataSetIterator baseIterator, int[] splits) {
int totalBatches = 0;
for (val v:splits)
totalBatches += v;
if (totalBatches < 0)
throw new ND4JIllegalStateException("totalExamples number should be positive value");
if (!baseIterator.resetSupported())
throw new ND4JIllegalStateException("Underlying iterator doesn't support reset, so it can't be used for runtime-split");
this.backedIterator = baseIterator;
this.totalExamples = totalBatches;
this.ratio = 0.0;
this.numTrain = (long) (totalExamples * ratio);
this.numTest = totalExamples - numTrain;
this.ratios = null;
this.numArbitrarySets = splits.length;
this.splits = splits;
log.warn("IteratorSplitter is used: please ensure you don't use randomization/shuffle in underlying iterator!");
}
public List<MultiDataSetIterator> getIterators() {
List<MultiDataSetIterator> retVal = new ArrayList<>();
int partN = 0;
int bottom = 0;
for (final int split : splits) {
ScrollableMultiDataSetIterator partIterator =
new ScrollableMultiDataSetIterator(partN++, backedIterator, counter, firstTrain,
new int[]{bottom,split});
bottom += split;
retVal.add(partIterator);
}
return retVal;
}
/**
* This method returns train iterator instance
*
* @return
*/
@Deprecated
public MultiDataSetIterator getTrainIterator() {
return new MultiDataSetIterator() {
@Override
public MultiDataSet next(int num) {
throw new UnsupportedOperationException("To be implemented yet");
}
@Override
public void setPreProcessor(MultiDataSetPreProcessor preProcessor) {
backedIterator.setPreProcessor(preProcessor);
}
@Override
public MultiDataSetPreProcessor getPreProcessor() {
return backedIterator.getPreProcessor();
}
@Override
public boolean resetSupported() {
return backedIterator.resetSupported();
}
@Override
public boolean asyncSupported() {
return backedIterator.asyncSupported();
}
@Override
public void reset() {
resetPending.set(true);
}
@Override
public boolean hasNext() {
if (resetPending.get()) {
if (resetSupported()) {
backedIterator.reset();
counter.set(0);
resetPending.set(false);
} else
throw new UnsupportedOperationException("Reset isn't supported by underlying iterator");
}
val state = backedIterator.hasNext();
if (state && counter.get() < numTrain)
return true;
else
return false;
}
@Override
public MultiDataSet next() {
counter.incrementAndGet();
val p = backedIterator.next();
if (counter.get() == 1 && firstTrain == null) {
// first epoch ever, we'll save first dataset and will use it to check for equality later
firstTrain = (org.nd4j.linalg.dataset.MultiDataSet) p.copy();
firstTrain.detach();
} else if (counter.get() == 1) {
// epoch > 1, comparing first dataset to previously stored dataset. they should be equal
int cnt = 0;
for (val c: p.getFeatures())
if (!c.equalsWithEps(firstTrain.getFeatures()[cnt++], 1e-5))
throw new ND4JIllegalStateException("First examples do not match. Randomization was used?");
}
return p;
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
/**
* This method returns test iterator instance
*
* @return
*/
@Deprecated
public MultiDataSetIterator getTestIterator() {
return new MultiDataSetIterator() {
@Override
public MultiDataSet next(int num) {
throw new UnsupportedOperationException("To be implemented yet");
}
@Override
public void setPreProcessor(MultiDataSetPreProcessor preProcessor) {
backedIterator.setPreProcessor(preProcessor);
}
@Override
public MultiDataSetPreProcessor getPreProcessor() {
return backedIterator.getPreProcessor();
}
@Override
public boolean resetSupported() {
return backedIterator.resetSupported();
}
@Override
public boolean asyncSupported() {
return backedIterator.asyncSupported();
}
@Override
public void reset() {
resetPending.set(true);
}
@Override
public boolean hasNext() {
val state = backedIterator.hasNext();
if (state && counter.get() < numTrain + numTest)
return true;
else
return false;
}
@Override
public MultiDataSet next() {
counter.incrementAndGet();
return backedIterator.next();
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultiDataSetWrapperIterator.java
+122
View File
@@ -0,0 +1,122 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import org.nd4j.linalg.dataset.api.MultiDataSet;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
import java.util.List;
public class MultiDataSetWrapperIterator implements DataSetIterator {
protected MultiDataSetIterator iterator;
protected DataSetPreProcessor preProcessor;
/**
* @param iterator Undelying iterator to wrap
*/
public MultiDataSetWrapperIterator(MultiDataSetIterator iterator) {
this.iterator = iterator;
}
@Override
public DataSet next(int num) {
throw new UnsupportedOperationException();
}
@Override
public int inputColumns() {
throw new UnsupportedOperationException();
}
@Override
public int totalOutcomes() {
throw new UnsupportedOperationException();
}
@Override
public boolean resetSupported() {
return iterator.resetSupported();
}
@Override
public boolean asyncSupported() {
return iterator.asyncSupported();
}
@Override
public void reset() {
iterator.reset();
}
@Override
public int batch() {
throw new UnsupportedOperationException();
}
@Override
public void setPreProcessor(DataSetPreProcessor preProcessor) {
this.preProcessor = preProcessor;
}
@Override
public DataSetPreProcessor getPreProcessor() {
return preProcessor;
}
@Override
public List<String> getLabels() {
throw new UnsupportedOperationException();
}
@Override
public boolean hasNext() {
return iterator.hasNext();
}
@Override
public DataSet next() {
MultiDataSet mds = iterator.next();
if (mds.getFeatures().length > 1 || mds.getLabels().length > 1)
throw new UnsupportedOperationException(
"This iterator is able to convert MultiDataSet with number of inputs/outputs of 1");
INDArray features = mds.getFeatures()[0];
INDArray labels = mds.getLabels() != null ? mds.getLabels()[0] : features;
INDArray fMask = mds.getFeaturesMaskArrays() != null ? mds.getFeaturesMaskArrays()[0] : null;
INDArray lMask = mds.getLabelsMaskArrays() != null ? mds.getLabelsMaskArrays()[0] : null;
DataSet ds = new DataSet(features, labels, fMask, lMask);
if (preProcessor != null)
preProcessor.preProcess(ds);
return ds;
}
@Override
public void remove() {
// no-op
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultipleEpochsIterator.java
Executable
+262
View File
@@ -0,0 +1,262 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import lombok.Setter;
import org.nd4j.shade.guava.annotations.VisibleForTesting;
import org.nd4j.shade.guava.collect.Lists;
import lombok.Getter;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.concurrent.atomic.AtomicLong;
@Deprecated
public class MultipleEpochsIterator implements DataSetIterator {
@VisibleForTesting
@Getter
@Setter
protected int epochs = 0;
protected int numEpochs;
protected int batch = 0;
protected int lastBatch = batch;
protected DataSetIterator iter;
protected DataSet ds;
protected List<DataSet> batchedDS = Lists.newArrayList();
protected static final Logger log = LoggerFactory.getLogger(MultipleEpochsIterator.class);
@Getter
protected DataSetPreProcessor preProcessor;
protected boolean newEpoch = false;
protected AtomicLong iterationsCounter = new AtomicLong(0);
protected long totalIterations = Long.MAX_VALUE;
@Deprecated
public MultipleEpochsIterator(int numEpochs, DataSetIterator iter) {
this.numEpochs = numEpochs;
this.iter = iter;
}
@Deprecated
public MultipleEpochsIterator(int numEpochs, DataSetIterator iter, int queueSize) {
this.numEpochs = numEpochs;
this.iter = iter;
}
@Deprecated
public MultipleEpochsIterator(DataSetIterator iter, int queueSize, long totalIterations) {
this.numEpochs = Integer.MAX_VALUE;
this.iter = iter;
this.totalIterations = totalIterations;
}
@Deprecated
public MultipleEpochsIterator(DataSetIterator iter, long totalIterations) {
this.numEpochs = Integer.MAX_VALUE;
this.iter = iter;
this.totalIterations = totalIterations;
}
@Deprecated
public MultipleEpochsIterator(int numEpochs, DataSet ds) {
this.numEpochs = numEpochs;
this.ds = ds;
}
/**
* Like the standard next method but allows a
* customizable number of examples returned
*
* @param num the number of examples
* @return the next data applyTransformToDestination
*/
@Override
public DataSet next(int num) {
if (!hasNext()) {
throw new NoSuchElementException("No next element");
}
DataSet next;
batch++;
iterationsCounter.incrementAndGet();
if (iter == null) {
// return full DataSet
if (num == -1) {
next = ds;
if (epochs < numEpochs)
trackEpochs();
}
// return DataSet broken into batches
else {
if (batchedDS.isEmpty() && num > 0)
batchedDS = ds.batchBy(num);
next = batchedDS.get(batch);
if (batch + 1 == batchedDS.size()) {
trackEpochs();
if (epochs < numEpochs)
batch = -1;
}
}
} else {
next = (num == -1 ? iter.next() : iter.next(num));
if (next == null) {
throw new IllegalStateException("Iterator returned null DataSet");
}
if (!iter.hasNext()) {
trackEpochs();
// track number of epochs and won't reset if it's over
if (epochs < numEpochs) {
iter.reset();
lastBatch = batch;
batch = 0;
}
}
}
if (preProcessor != null)
preProcessor.preProcess(next);
return next;
}
public void trackEpochs() {
epochs++;
newEpoch = true;
}
@Override
public DataSet next() {
return next(-1);
}
/**
* Input columns for the dataset
*
* @return
*/
@Override
public int inputColumns() {
return iter.inputColumns();
}
/**
* The number of labels for the dataset
*
* @return
*/
@Override
public int totalOutcomes() {
return iter.totalOutcomes();
}
@Override
public boolean resetSupported() {
return iter.resetSupported();
}
@Override
public boolean asyncSupported() {
return iter.asyncSupported();
}
/**
* Resets the iterator back to the beginning
*/
@Override
public void reset() {
if (!iter.resetSupported()) {
throw new IllegalStateException(
"Cannot reset MultipleEpochsIterator with base iter that does not support reset");
}
epochs = 0;
lastBatch = batch;
batch = 0;
iterationsCounter.set(0);
iter.reset();
}
/**
* Batch size
*
* @return
*/
@Override
public int batch() {
return iter.batch();
}
@Override
public void setPreProcessor(DataSetPreProcessor preProcessor) {
this.preProcessor = preProcessor;
}
@Override
public List<String> getLabels() {
return iter.getLabels();
}
/**
* Returns {@code true} if the iteration has more elements.
* (In other words, returns {@code true} if {@link #next} would
* return an element rather than throwing an exception.)
*
* @return {@code true} if the iteration has more elements
*/
@Override
public boolean hasNext() {
if (iterationsCounter.get() >= totalIterations)
return false;
if (newEpoch) {
log.info("Epoch " + epochs + ", number of batches completed " + lastBatch);
newEpoch = false;
}
if (iter == null)
return (epochs < numEpochs) && ((!batchedDS.isEmpty() && batchedDS.size() > batch) || batchedDS.isEmpty());
else
// either there are still epochs to complete or its the first epoch
return (epochs < numEpochs) || (iter.hasNext() && (epochs == 0 || epochs == numEpochs));
}
/**
* Removes from the underlying collection the last element returned
* by this iterator (optional operation). This method can be called
* only once per call to {@link #next}. The behavior of an iterator
* is unspecified if the underlying collection is modified while the
* iteration is in progress in any way other than by calling this
* method.
*
* @throws UnsupportedOperationException if the {@code remove}
* operation is not supported by this iterator
* @throws IllegalStateException if the {@code next} method has not
* yet been called, or the {@code remove} method has already
* been called after the last call to the {@code next}
* method
*/
@Override
public void remove() {
iter.remove();
}
}
deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/RandomDataSetIterator.java
+62
View File
@@ -0,0 +1,62 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.datasets.iterator;
import org.nd4j.linalg.factory.Nd4j;
public class RandomDataSetIterator extends MultiDataSetWrapperIterator {
public enum Values {RANDOM_UNIFORM, RANDOM_NORMAL, ONE_HOT, ZEROS, ONES, BINARY, INTEGER_0_10, INTEGER_0_100, INTEGER_0_1000,
INTEGER_0_10000, INTEGER_0_100000;
public RandomMultiDataSetIterator.Values toMdsValues(){
return RandomMultiDataSetIterator.Values.valueOf(this.toString());
}
};
/**
* @param numMiniBatches Number of minibatches per epoch
* @param featuresShape Features shape
* @param labelsShape Labels shape
* @param featureValues Type of values for the features
* @param labelValues Type of values for the labels
*/
public RandomDataSetIterator(int numMiniBatches, long[] featuresShape, long[] labelsShape, Values featureValues, Values labelValues){
this(numMiniBatches, featuresShape, labelsShape, featureValues, labelValues, Nd4j.order(), Nd4j.order());
}
/**
* @param numMiniBatches Number of minibatches per epoch
* @param featuresShape Features shape
* @param labelsShape Labels shape
* @param featureValues Type of values for the features
* @param labelValues Type of values for the labels
* @param featuresOrder Array order ('c' or 'f') for the features array
* @param labelsOrder Array order ('c' or 'f') for the labels array
*/
public RandomDataSetIterator(int numMiniBatches, long[] featuresShape, long[] labelsShape, Values featureValues, Values labelValues,
char featuresOrder, char labelsOrder){
super(new RandomMultiDataSetIterator.Builder(numMiniBatches)
.addFeatures(featuresShape, featuresOrder, featureValues.toMdsValues())
.addLabels(labelsShape, labelsOrder, labelValues.toMdsValues())
.build());
}
}

Some files were not shown because too many files have changed in this diff Show More