wehub/deeplearning4j--deeplearning4j
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:47:05 +08:00
commit 4f3b7da785
7394 changed files with 2005594 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/datasets/test/TestDataSetIterator.java
+117
View File
@@ -0,0 +1,117 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.datasets.test;
import lombok.Getter;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import java.util.List;
public class TestDataSetIterator implements DataSetIterator {
/**
*
*/
private static final long serialVersionUID = -3042802726018263331L;
private DataSetIterator wrapped;
private int numDataSets = 0;
@Getter
private DataSetPreProcessor preProcessor;
public TestDataSetIterator(DataSetIterator wrapped) {
super();
this.wrapped = wrapped;
}
@Override
public boolean hasNext() {
return wrapped.hasNext();
}
@Override
public DataSet next() {
numDataSets++;
DataSet next = wrapped.next();
if (preProcessor != null)
preProcessor.preProcess(next);
return next;
}
@Override
public void remove() {
wrapped.remove();
}
@Override
public int inputColumns() {
return wrapped.inputColumns();
}
@Override
public int totalOutcomes() {
return wrapped.totalOutcomes();
}
@Override
public boolean resetSupported() {
return wrapped.resetSupported();
}
@Override
public boolean asyncSupported() {
return wrapped.asyncSupported();
}
@Override
public void reset() {
wrapped.reset();
}
@Override
public int batch() {
return wrapped.batch();
}
@Override
public void setPreProcessor(DataSetPreProcessor preProcessor) {
this.preProcessor = preProcessor;
}
@Override
public List<String> getLabels() {
return null;
}
public synchronized int getNumDataSets() {
return numDataSets;
}
@Override
public DataSet next(int num) {
return wrapped.next(num);
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/datasets/vectorizer/Vectorizer.java
+38
View File
@@ -0,0 +1,38 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.datasets.vectorizer;
import org.nd4j.linalg.dataset.DataSet;
import java.io.Serializable;
public interface Vectorizer extends Serializable {
/**
* Vectorizes the input source in to a dataset
* @return Adam Gibson
*/
DataSet vectorize();
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/evaluation/EvaluationTools.java
+345
View File
@@ -0,0 +1,345 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.evaluation;
import org.apache.commons.io.FileUtils;
import org.deeplearning4j.ui.api.Component;
import org.deeplearning4j.ui.api.LengthUnit;
import org.deeplearning4j.ui.components.chart.ChartHistogram;
import org.deeplearning4j.ui.components.chart.ChartLine;
import org.deeplearning4j.ui.components.chart.style.StyleChart;
import org.deeplearning4j.ui.components.component.ComponentDiv;
import org.deeplearning4j.ui.components.component.style.StyleDiv;
import org.deeplearning4j.ui.components.table.ComponentTable;
import org.deeplearning4j.ui.components.table.style.StyleTable;
import org.deeplearning4j.ui.components.text.ComponentText;
import org.deeplearning4j.ui.components.text.style.StyleText;
import org.deeplearning4j.ui.standalone.StaticPageUtil;
import org.nd4j.evaluation.classification.EvaluationCalibration;
import org.nd4j.evaluation.classification.ROC;
import org.nd4j.evaluation.classification.ROCMultiClass;
import org.nd4j.evaluation.curves.Histogram;
import org.nd4j.evaluation.curves.PrecisionRecallCurve;
import org.nd4j.evaluation.curves.ReliabilityDiagram;
import org.nd4j.evaluation.curves.RocCurve;
import java.awt.*;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
public class EvaluationTools {
private static final String ROC_TITLE = "ROC: TPR/Recall (y) vs. FPR (x)";
private static final String PR_TITLE = "Precision (y) vs. Recall (x)";
private static final String PR_THRESHOLD_TITLE = "Precision and Recall (y) vs. Classifier Threshold (x)";
private static final double CHART_WIDTH_PX = 600.0;
private static final double CHART_HEIGHT_PX = 400.0;
private static final StyleChart CHART_STYLE = new StyleChart.Builder().width(CHART_WIDTH_PX, LengthUnit.Px)
.height(CHART_HEIGHT_PX, LengthUnit.Px).margin(LengthUnit.Px, 60, 60, 75, 10).strokeWidth(2.0)
.seriesColors(Color.BLUE, Color.LIGHT_GRAY).build();
private static final StyleChart CHART_STYLE_PRECISION_RECALL =
new StyleChart.Builder().width(CHART_WIDTH_PX, LengthUnit.Px).height(CHART_HEIGHT_PX, LengthUnit.Px)
.margin(LengthUnit.Px, 60, 60, 40, 10).strokeWidth(2.0)
.seriesColors(Color.BLUE, Color.GREEN).build();
private static final StyleTable TABLE_STYLE = new StyleTable.Builder().backgroundColor(Color.WHITE)
.headerColor(Color.LIGHT_GRAY).borderWidth(1).columnWidths(LengthUnit.Percent, 50, 50)
.width(400, LengthUnit.Px).height(200, LengthUnit.Px).build();
private static final StyleDiv OUTER_DIV_STYLE = new StyleDiv.Builder().width(2 * CHART_WIDTH_PX, LengthUnit.Px)
.height(CHART_HEIGHT_PX, LengthUnit.Px).build();
private static final StyleDiv OUTER_DIV_STYLE_WIDTH_ONLY =
new StyleDiv.Builder().width(2 * CHART_WIDTH_PX, LengthUnit.Px).build();
private static final StyleDiv INNER_DIV_STYLE = new StyleDiv.Builder().width(CHART_WIDTH_PX, LengthUnit.Px)
.floatValue(StyleDiv.FloatValue.left).build();
private static final StyleDiv PAD_DIV_STYLE = new StyleDiv.Builder().width(CHART_WIDTH_PX, LengthUnit.Px)
.height(100, LengthUnit.Px).floatValue(StyleDiv.FloatValue.left).build();
private static final ComponentDiv PAD_DIV = new ComponentDiv(PAD_DIV_STYLE);
private static final StyleText HEADER_TEXT_STYLE =
new StyleText.Builder().color(Color.BLACK).fontSize(16).underline(true).build();
private static final StyleDiv HEADER_DIV_STYLE =
new StyleDiv.Builder().width(2 * CHART_WIDTH_PX - 150, LengthUnit.Px).height(30, LengthUnit.Px)
.backgroundColor(Color.LIGHT_GRAY).margin(LengthUnit.Px, 5, 5, 200, 10)
.floatValue(StyleDiv.FloatValue.left).build();
private static final StyleDiv HEADER_DIV_STYLE_1400 = new StyleDiv.Builder().width(1400 - 150, LengthUnit.Px)
.height(30, LengthUnit.Px).backgroundColor(Color.LIGHT_GRAY).margin(LengthUnit.Px, 5, 5, 200, 10)
.floatValue(StyleDiv.FloatValue.left).build();
private static final StyleDiv HEADER_DIV_PAD_STYLE = new StyleDiv.Builder().width(2 * CHART_WIDTH_PX, LengthUnit.Px)
.height(150, LengthUnit.Px).backgroundColor(Color.WHITE).build();
private static final StyleDiv HEADER_DIV_TEXT_PAD_STYLE =
new StyleDiv.Builder().width(120, LengthUnit.Px).height(30, LengthUnit.Px)
.backgroundColor(Color.LIGHT_GRAY).floatValue(StyleDiv.FloatValue.left).build();
private static final ComponentTable INFO_TABLE = new ComponentTable.Builder(
new StyleTable.Builder().backgroundColor(Color.WHITE).borderWidth(0).build())
.content(new String[][] {
{"Precision", "(true positives) / (true positives + false positives)"},
{"True Positive Rate (Recall)",
"(true positives) / (data positives)"},
{"False Positive Rate", "(false positives) / (data negatives)"}})
.build();
private EvaluationTools() {}
/**
* Given a {@link ROC} chart, export the ROC chart and precision vs. recall charts to a stand-alone HTML file
* @param roc ROC to export
* @param file File to export to
*/
public static void exportRocChartsToHtmlFile(ROC roc, File file) throws IOException {
String rocAsHtml = rocChartToHtml(roc);
FileUtils.writeStringToFile(file, rocAsHtml);
}
/**
* Given a {@link ROCMultiClass} chart, export the ROC chart and precision vs. recall charts to a stand-alone HTML file
* @param roc ROC to export
* @param file File to export to
*/
public static void exportRocChartsToHtmlFile(ROCMultiClass roc, File file) throws Exception {
String rocAsHtml = rocChartToHtml(roc);
FileUtils.writeStringToFile(file, rocAsHtml, StandardCharsets.UTF_8);
}
/**
* Given a {@link ROC} instance, render the ROC chart and precision vs. recall charts to a stand-alone HTML file (returned as a String)
* @param roc ROC to render
*/
public static String rocChartToHtml(ROC roc) {
RocCurve rocCurve = roc.getRocCurve();
Component c = getRocFromPoints(ROC_TITLE, rocCurve, roc.getCountActualPositive(), roc.getCountActualNegative(),
roc.calculateAUC(), roc.calculateAUCPR());
Component c2 = getPRCharts(PR_TITLE, PR_THRESHOLD_TITLE, roc.getPrecisionRecallCurve());
return StaticPageUtil.renderHTML(c, c2);
}
/**
* Given a {@link ROCMultiClass} instance, render the ROC chart and precision vs. recall charts to a stand-alone HTML file (returned as a String)
* @param rocMultiClass ROC to render
*/
public static String rocChartToHtml(ROCMultiClass rocMultiClass) {
return rocChartToHtml(rocMultiClass, null);
}
/**
* Given a {@link ROCMultiClass} instance and (optionally) names for each class, render the ROC chart to a stand-alone
* HTML file (returned as a String)
* @param rocMultiClass ROC to render
* @param classNames Names of the classes. May be null
*/
public static String rocChartToHtml(ROCMultiClass rocMultiClass, List<String> classNames) {
int n = rocMultiClass.getNumClasses();
List<Component> components = new ArrayList<>(n);
for (int i = 0; i < n; i++) {
RocCurve roc = rocMultiClass.getRocCurve(i);
String headerText = "Class " + i;
if (classNames != null && classNames.size() > i) {
headerText += " (" + classNames.get(i) + ")";
}
headerText += " vs. All";;
Component headerDivPad = new ComponentDiv(HEADER_DIV_PAD_STYLE);
components.add(headerDivPad);
Component headerDivLeft = new ComponentDiv(HEADER_DIV_TEXT_PAD_STYLE);
Component headerDiv = new ComponentDiv(HEADER_DIV_STYLE, new ComponentText(headerText, HEADER_TEXT_STYLE));
Component c = getRocFromPoints(ROC_TITLE, roc, rocMultiClass.getCountActualPositive(i),
rocMultiClass.getCountActualNegative(i), rocMultiClass.calculateAUC(i),
rocMultiClass.calculateAUCPR(i));
Component c2 = getPRCharts(PR_TITLE, PR_THRESHOLD_TITLE, rocMultiClass.getPrecisionRecallCurve(i));
components.add(headerDivLeft);
components.add(headerDiv);
components.add(c);
components.add(c2);
}
return StaticPageUtil.renderHTML(components);
}
/**
* Given a {@link EvaluationCalibration} instance, export the charts to a stand-alone HTML file
* @param ec EvaluationCalibration instance to export HTML charts for
* @param file File to export to
*/
public static void exportevaluationCalibrationToHtmlFile(EvaluationCalibration ec, File file) throws IOException {
String asHtml = evaluationCalibrationToHtml(ec);
FileUtils.writeStringToFile(file, asHtml);
}
public static String evaluationCalibrationToHtml(EvaluationCalibration ec) {
List<Component> components = new ArrayList<>();
int nClasses = ec.numClasses();
//Distribution of class labels + distribution of predicted classes
Component headerDiv = new ComponentDiv(HEADER_DIV_STYLE_1400,
new ComponentText(
"Labels and Network Prediction Class Distributions (X: Class Index. Y: Count)",
HEADER_TEXT_STYLE));
components.add(headerDiv);
int[] labelCounts = ec.getLabelCountsEachClass();
int[] predictedCounts = ec.getPredictionCountsEachClass();
ChartHistogram.Builder chbLabels = new ChartHistogram.Builder("Label Class Distribution", CHART_STYLE);
ChartHistogram.Builder chbPredictions = new ChartHistogram.Builder("Predicted Class Distribution", CHART_STYLE);
for (int i = 0; i < nClasses; i++) {
double lower = i - 0.5;
double upper = i + 0.5;
chbLabels.addBin(lower, upper, labelCounts[i]);
chbPredictions.addBin(lower, upper, predictedCounts[i]);
}
ChartHistogram chL = chbLabels.build();
ChartHistogram chP = chbPredictions.build();
components.add(new ComponentDiv(OUTER_DIV_STYLE_WIDTH_ONLY, chL, chP));
//Reliability diagram, for each class
headerDiv = new ComponentDiv(HEADER_DIV_STYLE_1400, new ComponentText(
"Reliability Diagrams (X: Mean Predicted Value. Y: Fraction Positives)", HEADER_TEXT_STYLE));
components.add(headerDiv);
List<Component> sectionDiv = new ArrayList<>();
double[] zeroOne = new double[] {0.0, 1.0};
for (int i = 0; i < nClasses; i++) {
ReliabilityDiagram rd = ec.getReliabilityDiagram(i);
double[] x = rd.getMeanPredictedValueX();
double[] y = rd.getFractionPositivesY();
String title = rd.getTitle();
ChartLine cl = new ChartLine.Builder(title, CHART_STYLE).addSeries("Classifier", x, y)
.addSeries("Ideal Classifier", zeroOne, zeroOne).build();
sectionDiv.add(cl);
}
components.add(new ComponentDiv(OUTER_DIV_STYLE_WIDTH_ONLY, sectionDiv));
//Residual plots
headerDiv = new ComponentDiv(HEADER_DIV_STYLE_1400, new ComponentText(
"Network Predictions - Residual Plots - |Label(i) - P(class(i))|", HEADER_TEXT_STYLE));
components.add(headerDiv);
sectionDiv = new ArrayList<>();
Histogram resPlotAll = ec.getResidualPlotAllClasses();
sectionDiv.add(getHistogram(resPlotAll));
for (int i = 0; i < nClasses; i++) {
Histogram resPlotCurrent = ec.getResidualPlot(i);
sectionDiv.add(getHistogram(resPlotCurrent));
}
components.add(new ComponentDiv(OUTER_DIV_STYLE_WIDTH_ONLY, sectionDiv));
//Histogram of probabilities, overall and for each class
headerDiv = new ComponentDiv(HEADER_DIV_STYLE_1400, new ComponentText(
"Network Prediction Probabilities (X: P(class). Y: Count)", HEADER_TEXT_STYLE));
components.add(headerDiv);
sectionDiv = new ArrayList<>();
Histogram allProbs = ec.getProbabilityHistogramAllClasses();
sectionDiv.add(getHistogram(allProbs));
for (int i = 0; i < nClasses; i++) {
Histogram classProbs = ec.getProbabilityHistogram(i);
sectionDiv.add(getHistogram(classProbs));
}
components.add(new ComponentDiv(OUTER_DIV_STYLE_WIDTH_ONLY, sectionDiv));
return StaticPageUtil.renderHTML(components);
}
private static Component getRocFromPoints(String title, RocCurve roc, long positiveCount, long negativeCount,
double auc, double aucpr) {
double[] zeroOne = new double[] {0.0, 1.0};
ChartLine chartLine = new ChartLine.Builder(title, CHART_STYLE).setXMin(0.0).setXMax(1.0).setYMin(0.0)
.setYMax(1.0).addSeries("ROC", roc.getX(), roc.getY()).addSeries("", zeroOne, zeroOne).build();
ComponentTable ct = new ComponentTable.Builder(TABLE_STYLE).header("Field", "Value")
.content(new String[][] {{"AUROC: Area under ROC:", String.format("%.5f", auc)},
{"AUPRC: Area under P/R:", String.format("%.5f", aucpr)},
{"Total Data Positive Count", String.valueOf(positiveCount)},
{"Total Data Negative Count", String.valueOf(negativeCount)}})
.build();
ComponentDiv divLeft = new ComponentDiv(INNER_DIV_STYLE, PAD_DIV, ct, PAD_DIV, INFO_TABLE);
ComponentDiv divRight = new ComponentDiv(INNER_DIV_STYLE, chartLine);
return new ComponentDiv(OUTER_DIV_STYLE, divLeft, divRight);
}
private static Component getPRCharts(String precisionRecallTitle, String prThresholdTitle,
PrecisionRecallCurve prCurve) {
ComponentDiv divLeft =
new ComponentDiv(INNER_DIV_STYLE, getPrecisionRecallCurve(precisionRecallTitle, prCurve));
ComponentDiv divRight =
new ComponentDiv(INNER_DIV_STYLE, getPrecisionRecallVsThreshold(prThresholdTitle, prCurve));
return new ComponentDiv(OUTER_DIV_STYLE, divLeft, divRight);
}
private static Component getPrecisionRecallCurve(String title, PrecisionRecallCurve prCurve) {
double[] recallX = prCurve.getRecall();
double[] precisionY = prCurve.getPrecision();
return new ChartLine.Builder(title, CHART_STYLE).setXMin(0.0).setXMax(1.0).setYMin(0.0).setYMax(1.0)
.addSeries("P vs R", recallX, precisionY).build();
}
private static Component getPrecisionRecallVsThreshold(String title, PrecisionRecallCurve prCurve) {
double[] recallY = prCurve.getRecall();
double[] precisionY = prCurve.getPrecision();
double[] thresholdX = prCurve.getThreshold();
return new ChartLine.Builder(title, CHART_STYLE_PRECISION_RECALL).setXMin(0.0).setXMax(1.0).setYMin(0.0)
.setYMax(1.0).addSeries("Precision", thresholdX, precisionY)
.addSeries("Recall", thresholdX, recallY).showLegend(true).build();
}
private static Component getHistogram(Histogram histogram) {
ChartHistogram.Builder chb = new ChartHistogram.Builder(histogram.getTitle(), CHART_STYLE);
double[] lower = histogram.getBinLowerBounds();
double[] upper = histogram.getBinUpperBounds();
int[] counts = histogram.getBinCounts();
for (int i = 0; i < counts.length; i++) {
chb.addBin(lower[i], upper[i], counts[i]);
}
return chb.build();
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/DeviceMetric.java
+45
View File
@@ -0,0 +1,45 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.listener;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import java.io.Serializable;
@Data
@Builder
@AllArgsConstructor
public class DeviceMetric implements Serializable {
private double load;
private double totalMemory;
private String deviceName;
private double temp;
private double memAvailable;
private long bandwidthDeviceToHost,bandwidthHostToDevice,bandwidthDeviceToDevice;
private DeviceMetric(){
//No-arg constructor for JSON/YAML
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/DiskInfo.java
+39
View File
@@ -0,0 +1,39 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.listener;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import java.io.Serializable;
@Data
@Builder
@AllArgsConstructor
public class DiskInfo implements Serializable {
private long bytesRead,bytesWritten,transferTime;
private String name,modelName;
private DiskInfo(){
//No-arg for JSON/YAML
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/HardwareMetric.java
+187
View File
@@ -0,0 +1,187 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.listener;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NonNull;
import org.nd4j.linalg.api.environment.Nd4jEnvironment;
import org.nd4j.linalg.api.ops.performance.PerformanceTracker;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.api.memory.MemcpyDirection;
import org.nd4j.shade.jackson.databind.ObjectMapper;
import org.nd4j.shade.jackson.dataformat.yaml.YAMLFactory;
import oshi.json.SystemInfo;
import oshi.json.hardware.CentralProcessor;
import oshi.json.hardware.GlobalMemory;
import oshi.json.hardware.HWDiskStore;
import oshi.json.software.os.NetworkParams;
import oshi.util.Util;
import java.io.IOException;
import java.io.Serializable;
import java.util.*;
@Builder
@Data
@AllArgsConstructor
public class HardwareMetric implements Serializable {
private static ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory());
private Map<Integer,DeviceMetric> perCoreMetrics;
private long physicalProcessorCount,logicalProcessorCount;
private long currentMemoryUse;
private Map<Integer,DeviceMetric> gpuMetrics;
private String hostName;
private long ioWaitTime;
private long averagedCpuLoad;
private Map<Integer,DiskInfo> diskInfo;
private String name;
private HardwareMetric(){
//No-arg for JSON/YAML
}
/**
* Runs {@link #fromSystem(SystemInfo)}
* with a fresh {@link SystemInfo}
* @return the hardware metric based on
* the current snapshot of the system this
* runs on
*/
public static HardwareMetric fromSystem() {
return fromSystem(new SystemInfo());
}
/**
* Returns the relevant information
* needed for system diagnostics
* based on the {@link SystemInfo}
* @param systemInfo the system info to use
* @return the {@link HardwareMetric} for the
* system this process runs on
*/
public static HardwareMetric fromSystem(SystemInfo systemInfo) {
return fromSystem(systemInfo,UUID.randomUUID().toString());
}
/**
* Returns the relevant information
* needed for system diagnostics
* based on the {@link SystemInfo}
* @param systemInfo the system info to use
* @return the {@link HardwareMetric} for the
* system this process runs on
*/
public static HardwareMetric fromSystem(SystemInfo systemInfo,String name) {
HardwareMetricBuilder builder = HardwareMetric.builder();
CentralProcessor processor = systemInfo.getHardware().getProcessor();
long[] prevTicks = processor.getSystemCpuLoadTicks();
// Wait a second...
Util.sleep(1000);
long[] ticks = processor.getSystemCpuLoadTicks();
long iowait = ticks[oshi.hardware.CentralProcessor.TickType.IOWAIT.getIndex()] - prevTicks[oshi.hardware.CentralProcessor.TickType.IOWAIT.getIndex()];
GlobalMemory globalMemory = systemInfo.getHardware().getMemory();
NetworkParams networkParams = systemInfo.getOperatingSystem().getNetworkParams();
double[] processorCpuLoadBetweenTicks = processor.getProcessorCpuLoadBetweenTicks();
Map<Integer,DeviceMetric> cpuMetrics = new LinkedHashMap<>();
for(int i = 0; i < processorCpuLoadBetweenTicks.length; i++) {
cpuMetrics.put(i, DeviceMetric.builder()
.load(processorCpuLoadBetweenTicks[i]).
build());
}
Map<Integer,DiskInfo> diskInfoMap = new LinkedHashMap<>();
HWDiskStore[] diskStores = systemInfo.getHardware().getDiskStores();
for(int i = 0; i < diskStores.length; i++) {
HWDiskStore diskStore = diskStores[i];
DiskInfo diskInfo = DiskInfo.builder()
.bytesRead(diskStore.getReadBytes())
.bytesWritten(diskStore.getWriteBytes())
.name(diskStore.getName())
.modelName(diskStore.getModel())
.transferTime(diskStore.getTransferTime())
.build();
diskInfoMap.put(i,diskInfo);
}
Map<Integer,DeviceMetric> gpuMetric = new HashMap<>();
if(Nd4j.getBackend().getClass().getName().toLowerCase().contains("cublas")) {
Properties info = Nd4j.getExecutioner().getEnvironmentInformation();
/**
*
*/
List<Map<String, Object>> devicesList = (List<Map<String, Object>>) info.get(Nd4jEnvironment.CUDA_DEVICE_INFORMATION_KEY);
for(int i = 0; i < devicesList.size(); i++) {
double available = Double.parseDouble(devicesList.get(i).get(Nd4jEnvironment.CUDA_FREE_MEMORY_KEY).toString());
Map<MemcpyDirection, Long> memcpyDirectionLongMap = PerformanceTracker.getInstance().getCurrentBandwidth().get(i);
DeviceMetric deviceMetric = DeviceMetric.builder()
.bandwidthHostToDevice(memcpyDirectionLongMap.get(MemcpyDirection.HOST_TO_DEVICE))
.bandwidthDeviceToHost(memcpyDirectionLongMap.get(MemcpyDirection.DEVICE_TO_HOST))
.bandwidthDeviceToDevice(memcpyDirectionLongMap.get(MemcpyDirection.DEVICE_TO_DEVICE))
.memAvailable(available).totalMemory(Double.parseDouble(devicesList.get(i).get(Nd4jEnvironment.CUDA_TOTAL_MEMORY_KEY).toString()))
.deviceName(devicesList.get(i).get(Nd4jEnvironment.CUDA_DEVICE_NAME_KEY).toString())
.build();
gpuMetric.put(i,deviceMetric);
}
}
return builder.logicalProcessorCount(processor.getLogicalProcessorCount())
.physicalProcessorCount(processor.getPhysicalProcessorCount())
.name(name)
.averagedCpuLoad((long)(processor.getSystemCpuLoad() * 100))
.ioWaitTime(iowait).gpuMetrics(gpuMetric)
.hostName(networkParams.getHostName()).diskInfo(diskInfoMap)
.currentMemoryUse(globalMemory.getTotal() - globalMemory.getAvailable())
.perCoreMetrics(cpuMetrics)
.build();
}
public String toYaml(){
try {
return yamlMapper.writeValueAsString(this);
} catch (Exception e){
throw new RuntimeException(e);
}
}
public static HardwareMetric fromYaml(@NonNull String yaml){
try {
return yamlMapper.readValue(yaml, HardwareMetric.class);
} catch (IOException e){
throw new RuntimeException(e);
}
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/SystemInfoFilePrintListener.java
+132
View File
@@ -0,0 +1,132 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.listener;
import lombok.NonNull;
import lombok.Builder;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.deeplearning4j.nn.api.Model;
import org.deeplearning4j.optimize.api.TrainingListener;
import org.nd4j.linalg.api.ndarray.INDArray;
import oshi.json.SystemInfo;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
@Slf4j
@Builder
public class SystemInfoFilePrintListener implements TrainingListener {
private boolean printOnEpochStart;
private boolean printOnEpochEnd;
private boolean printOnForwardPass;
private boolean printOnBackwardPass;
private boolean printOnGradientCalculation;
private File printFileTarget;
public SystemInfoFilePrintListener(boolean printOnEpochStart, boolean printOnEpochEnd, boolean printOnForwardPass, boolean printOnBackwardPass, boolean printOnGradientCalculation, @NonNull File printFileTarget) {
this.printOnEpochStart = printOnEpochStart;
this.printOnEpochEnd = printOnEpochEnd;
this.printOnForwardPass = printOnForwardPass;
this.printOnBackwardPass = printOnBackwardPass;
this.printOnGradientCalculation = printOnGradientCalculation;
this.printFileTarget = printFileTarget;
}
@Override
public void iterationDone(Model model, int iteration, int epoch) {
}
@Override
public void onEpochStart(Model model) {
if(!printOnEpochStart || printFileTarget == null)
return;
writeFileWithMessage("epoch end");
}
@Override
public void onEpochEnd(Model model) {
if(!printOnEpochEnd || printFileTarget == null)
return;
writeFileWithMessage("epoch begin");
}
@Override
public void onForwardPass(Model model, List<INDArray> activations) {
if(!printOnBackwardPass || printFileTarget == null)
return;
writeFileWithMessage("forward pass");
}
@Override
public void onForwardPass(Model model, Map<String, INDArray> activations) {
if(!printOnForwardPass || printFileTarget == null)
return;
writeFileWithMessage("forward pass");
}
@Override
public void onGradientCalculation(Model model) {
if(!printOnGradientCalculation || printFileTarget == null)
return;
writeFileWithMessage("gradient calculation");
}
@Override
public void onBackwardPass(Model model) {
if(!printOnBackwardPass || printFileTarget == null)
return;
writeFileWithMessage("backward pass");
}
private void writeFileWithMessage(String status) {
if(printFileTarget == null) {
log.warn("File not specified for writing!");
}
SystemInfo systemInfo = new SystemInfo();
log.info("Writing system info to file on " + status + ": " + printFileTarget.getAbsolutePath());
try {
FileUtils.write(printFileTarget,systemInfo.toPrettyJSON(), true);
} catch (IOException e) {
log.error("Error writing file for system info",e);
}
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/SystemInfoPrintListener.java
+114
View File
@@ -0,0 +1,114 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.listener;
import lombok.Builder;
import lombok.extern.slf4j.Slf4j;
import org.deeplearning4j.nn.api.Model;
import org.deeplearning4j.optimize.api.TrainingListener;
import org.nd4j.linalg.api.ndarray.INDArray;
import oshi.json.SystemInfo;
import java.util.List;
import java.util.Map;
/**
* Using {@link SystemInfo} - it logs a json representation
* of system info using slf4j.
*
* @author Adam Gibson
*/
@Slf4j
@Builder
public class SystemInfoPrintListener implements TrainingListener {
private boolean printOnEpochStart;
private boolean printOnEpochEnd;
private boolean printOnForwardPass;
private boolean printOnBackwardPass;
private boolean printOnGradientCalculation;
private static final String SYSTEM_INFO = "System info on epoch end: ";
@Override
public void iterationDone(Model model, int iteration, int epoch) {
}
@Override
public void onEpochStart(Model model) {
if(!printOnEpochStart)
return;
SystemInfo systemInfo = new SystemInfo();
log.info("System info on epoch begin: ");
log.info(systemInfo.toPrettyJSON());
}
@Override
public void onEpochEnd(Model model) {
if(!printOnEpochEnd)
return;
SystemInfo systemInfo = new SystemInfo();
log.info(SYSTEM_INFO);
log.info(systemInfo.toPrettyJSON());
}
@Override
public void onForwardPass(Model model, List<INDArray> activations) {
if(!printOnBackwardPass)
return;
SystemInfo systemInfo = new SystemInfo();
log.info(SYSTEM_INFO);
log.info(systemInfo.toPrettyJSON());
}
@Override
public void onForwardPass(Model model, Map<String, INDArray> activations) {
if(!printOnForwardPass)
return;
SystemInfo systemInfo = new SystemInfo();
log.info(SYSTEM_INFO);
log.info(systemInfo.toPrettyJSON());
}
@Override
public void onGradientCalculation(Model model) {
if(!printOnGradientCalculation)
return;
SystemInfo systemInfo = new SystemInfo();
log.info(SYSTEM_INFO);
log.info(systemInfo.toPrettyJSON());
}
@Override
public void onBackwardPass(Model model) {
if(!printOnBackwardPass)
return;
SystemInfo systemInfo = new SystemInfo();
log.info(SYSTEM_INFO);
log.info(systemInfo.toPrettyJSON());
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/SystemPolling.java
+142
View File
@@ -0,0 +1,142 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.listener;
import lombok.extern.slf4j.Slf4j;
import org.nd4j.shade.jackson.databind.ObjectMapper;
import org.nd4j.shade.jackson.dataformat.yaml.YAMLFactory;
import oshi.json.SystemInfo;
import java.io.File;
import java.io.IOException;
import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@Slf4j
public class SystemPolling {
private ScheduledExecutorService scheduledExecutorService;
private long pollEveryMillis;
private File outputDirectory;
private NameProvider nameProvider;
private ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
private SystemPolling(long pollEveryMillis,File outputDirectory,NameProvider nameProvider) {
this.pollEveryMillis = pollEveryMillis;
this.outputDirectory = outputDirectory;
this.nameProvider = nameProvider;
}
/**
* Run the polling in the background as a thread pool
* running every {@link #pollEveryMillis} milliseconds
*/
public void run() {
scheduledExecutorService = Executors.newScheduledThreadPool(1);
scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
SystemInfo systemInfo = new SystemInfo();
HardwareMetric hardwareMetric = HardwareMetric.fromSystem(systemInfo,nameProvider.nextName());
File hardwareFile = new File(outputDirectory,hardwareMetric.getName() + ".yml");
try {
objectMapper.writeValue(hardwareFile,hardwareMetric);
} catch (IOException e) {
log.error("",e);
}
}
},0,pollEveryMillis, TimeUnit.MILLISECONDS);
}
/**
* Shut down the background polling
*/
public void stopPolling() {
scheduledExecutorService.shutdownNow();
}
/**
* The naming sequence provider.
* This is for allowing generation of naming the output
* according to some semantic sequence (such as a neural net epoch
* or some time stamp)
*/
public interface NameProvider {
String nextName();
}
public static class Builder {
private long pollEveryMillis;
private File outputDirectory;
private NameProvider nameProvider = new NameProvider() {
@Override
public String nextName() {
return UUID.randomUUID().toString();
}
};
/**
* The name provider for this {@link SystemPolling}
* the default value for this is a simple UUID
* @param nameProvider the name provider to use
* @return
*/
public Builder nameProvider(NameProvider nameProvider) {
this.nameProvider = nameProvider;
return this;
}
/**
* The interval in milliseconds in which to poll
* the system for diagnostics
* @param pollEveryMillis the interval in milliseconds
* @return
*/
public Builder pollEveryMillis(long pollEveryMillis) {
this.pollEveryMillis = pollEveryMillis;
return this;
}
/**
* The output directory for the files
* @param outputDirectory the output directory for the logs
* @return
*/
public Builder outputDirectory(File outputDirectory) {
this.outputDirectory = outputDirectory;
return this;
}
public SystemPolling build() {
return new SystemPolling(pollEveryMillis,outputDirectory,nameProvider);
}
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/DataSetLoader.java
+29
View File
@@ -0,0 +1,29 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.loader;
import org.nd4j.common.loader.Loader;
import org.nd4j.common.loader.Source;
import org.nd4j.linalg.dataset.DataSet;
public interface DataSetLoader extends Loader<DataSet> {
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/MultiDataSetLoader.java
+29
View File
@@ -0,0 +1,29 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.loader;
import org.nd4j.common.loader.Loader;
import org.nd4j.common.loader.Source;
import org.nd4j.linalg.dataset.api.MultiDataSet;
public interface MultiDataSetLoader extends Loader<MultiDataSet> {
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/impl/RecordReaderFileBatchLoader.java
+111
View File
@@ -0,0 +1,111 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.loader.impl;
import lombok.Getter;
import lombok.Setter;
import org.datavec.api.records.reader.RecordReader;
import org.datavec.api.records.reader.impl.filebatch.FileBatchRecordReader;
import org.deeplearning4j.core.loader.DataSetLoader;
import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator;
import org.nd4j.common.loader.FileBatch;
import org.nd4j.common.loader.Source;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
import java.io.IOException;
public class RecordReaderFileBatchLoader implements DataSetLoader {
private final RecordReader recordReader;
private final int batchSize;
private final int labelIndexFrom;
private final int labelIndexTo;
private final int numPossibleLabels;
private final boolean regression;
@Getter @Setter
private DataSetPreProcessor preProcessor;
/**
* Main constructor for classification. This will convert the input class index (at position labelIndex, with integer
* values 0 to numPossibleLabels-1 inclusive) to the appropriate one-hot output/labels representation.
*
* @param recordReader RecordReader: provides the source of the data
* @param batchSize Batch size (number of examples) for the output DataSet objects
* @param labelIndex Index of the label Writable (usually an IntWritable), as obtained by recordReader.next()
* @param numClasses Number of classes (possible labels) for classification
*/
public RecordReaderFileBatchLoader(RecordReader recordReader, int batchSize, int labelIndex, int numClasses) {
this(recordReader, batchSize, labelIndex, labelIndex, numClasses, false, null);
}
/**
* Main constructor for multi-label regression (i.e., regression with multiple outputs). Can also be used for single
* output regression with labelIndexFrom == labelIndexTo
*
* @param recordReader RecordReader to get data from
* @param labelIndexFrom Index of the first regression target
* @param labelIndexTo Index of the last regression target, inclusive
* @param batchSize Minibatch size
* @param regression Require regression = true. Mainly included to avoid clashing with other constructors previously defined :/
*/
public RecordReaderFileBatchLoader(RecordReader recordReader, int batchSize, int labelIndexFrom, int labelIndexTo,
boolean regression) {
this(recordReader, batchSize, labelIndexFrom, labelIndexTo, -1, regression, null);
}
/**
* Main constructor
*
* @param recordReader the recordreader to use
* @param batchSize Minibatch size - number of examples returned for each call of .next()
* @param labelIndexFrom the index of the label (for classification), or the first index of the labels for multi-output regression
* @param labelIndexTo only used if regression == true. The last index <i>inclusive</i> of the multi-output regression
* @param numPossibleLabels the number of possible labels for classification. Not used if regression == true
* @param regression if true: regression. If false: classification (assume labelIndexFrom is the class it belongs to)
* @param preProcessor Optional DataSetPreProcessor. May be null.
*/
public RecordReaderFileBatchLoader(RecordReader recordReader, int batchSize, int labelIndexFrom, int labelIndexTo,
int numPossibleLabels, boolean regression, DataSetPreProcessor preProcessor) {
this.recordReader = recordReader;
this.batchSize = batchSize;
this.labelIndexFrom = labelIndexFrom;
this.labelIndexTo = labelIndexTo;
this.numPossibleLabels = numPossibleLabels;
this.regression = regression;
this.preProcessor = preProcessor;
}
@Override
public DataSet load(Source source) throws IOException {
FileBatch fb = FileBatch.readFromZip(source.getInputStream());
//Wrap file batch in RecordReader
//Create RecordReaderDataSetIterator
//Return dataset
RecordReader rr = new FileBatchRecordReader(recordReader, fb);
RecordReaderDataSetIterator iter = new RecordReaderDataSetIterator(rr, null, batchSize, labelIndexFrom, labelIndexTo, numPossibleLabels, -1, regression);
if (preProcessor != null) {
iter.setPreProcessor(preProcessor);
}
DataSet ds = iter.next();
return ds;
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/impl/SerializedDataSetLoader.java
+39
View File
@@ -0,0 +1,39 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.loader.impl;
import org.deeplearning4j.core.loader.DataSetLoader;
import org.nd4j.common.loader.Source;
import org.nd4j.linalg.dataset.DataSet;
import java.io.IOException;
import java.io.InputStream;
public class SerializedDataSetLoader implements DataSetLoader {
@Override
public DataSet load(Source source) throws IOException {
DataSet ds = new DataSet();
try(InputStream is = source.getInputStream()){
ds.load(is);
}
return ds;
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/impl/SerializedMultiDataSetLoader.java
+39
View File
@@ -0,0 +1,39 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.loader.impl;
import org.deeplearning4j.core.loader.MultiDataSetLoader;
import org.nd4j.common.loader.Source;
import org.nd4j.linalg.dataset.api.MultiDataSet;
import java.io.IOException;
import java.io.InputStream;
public class SerializedMultiDataSetLoader implements MultiDataSetLoader {
@Override
public MultiDataSet load(Source source) throws IOException {
org.nd4j.linalg.dataset.MultiDataSet ds = new org.nd4j.linalg.dataset.MultiDataSet();
try(InputStream is = source.getInputStream()){
ds.load(is);
}
return ds;
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/parallelism/AsyncIterator.java
+141
View File
@@ -0,0 +1,141 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.parallelism;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import java.util.Iterator;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
@Slf4j
public class AsyncIterator<T extends Object> implements Iterator<T> {
@Getter
protected BlockingQueue<T> buffer;
protected ReaderThread<T> thread;
protected Iterator<T> iterator;
@Getter
protected T terminator = (T) new Object();
protected T nextElement;
protected AtomicBoolean shouldWork = new AtomicBoolean(true);
public AsyncIterator(@NonNull Iterator<T> iterator, int bufferSize) {
this.buffer = new LinkedBlockingQueue<>(bufferSize);
this.iterator = iterator;
thread = new ReaderThread<>(iterator, this.buffer, terminator);
thread.start();
}
public AsyncIterator(@NonNull Iterator<T> iterator) {
this(iterator, 1024);
}
@Override
public boolean hasNext() {
try {
if (nextElement != null && nextElement != terminator) {
return true;
}
// if on previous run we've got terminator - just return false
if (nextElement == terminator)
return false;
nextElement = buffer.take();
// same on this run
return (nextElement != terminator);
} catch (Exception e) {
log.error("Premature end of loop!");
return false;
}
}
@Override
public T next() {
T temp = nextElement;
nextElement = temp == terminator ? terminator : null;
return temp;
}
@Override
public void remove() {
// no-op
}
public void shutdown() {
if (shouldWork.get()) {
shouldWork.set(false);
thread.interrupt();
try {
// Shutdown() should be a synchronous operation since the iterator is reset after shutdown() is
// called in AsyncLabelAwareIterator.reset().
thread.join();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
nextElement = terminator;
buffer.clear();
}
}
private class ReaderThread<T> extends Thread implements Runnable {
private BlockingQueue<T> buffer;
private Iterator<T> iterator;
private T terminator;
public ReaderThread(Iterator<T> iterator, BlockingQueue<T> buffer, T terminator) {
this.buffer = buffer;
this.iterator = iterator;
this.terminator = terminator;
setDaemon(true);
setName("AsyncIterator Reader thread");
}
@Override
public void run() {
//log.info("AsyncReader [{}] started", Thread.currentThread().getId());
try {
while (iterator.hasNext() && shouldWork.get()) {
T smth = iterator.next();
if (smth != null)
buffer.put(smth);
}
buffer.put(terminator);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
// do nothing
shouldWork.set(false);
} catch (Exception e) {
// TODO: pass that forward
throw new RuntimeException(e);
} finally {
//log.info("AsyncReader [{}] stopped", Thread.currentThread().getId());
}
}
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/Persistable.java
+100
View File
@@ -0,0 +1,100 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.nio.ByteBuffer;
public interface Persistable extends Serializable {
/**
* Get the session id
* @return
*/
String getSessionID();
/**
* Get the type id
* @return
*/
String getTypeID();
/**
* Get the worker id
* @return
*/
String getWorkerID();
/**
* Get when this was created.
* @return
*/
long getTimeStamp();
//SerDe methods:
/**
* Length of the encoding, in bytes, when using {@link #encode()}
* Length may be different using {@link #encode(OutputStream)}, due to things like stream headers
* @return
*/
int encodingLengthBytes();
byte[] encode();
/**
* Encode this persistable in to a {@link ByteBuffer}
* @param buffer
*/
void encode(ByteBuffer buffer);
/**
* Encode this persistable in to an output stream
* @param outputStream
* @throws IOException
*/
void encode(OutputStream outputStream) throws IOException;
/**
* Decode the content of the given
* byte array in to this persistable
* @param decode
*/
void decode(byte[] decode);
/**
* Decode from the given {@link ByteBuffer}
* @param buffer
*/
void decode(ByteBuffer buffer);
/**
* Decode from the given input stream
* @param inputStream
* @throws IOException
*/
void decode(InputStream inputStream) throws IOException;
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorage.java
+220
View File
@@ -0,0 +1,220 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
import java.io.IOException;
import java.util.List;
public interface StatsStorage extends StatsStorageRouter {
/**
* Close any open resources (files, etc)
*/
void close() throws IOException;
/**
* @return Whether the StatsStorage implementation has been closed or not
*/
boolean isClosed();
/**
* Get a list of all sessions stored by this storage backend
*/
List<String> listSessionIDs();
/**
* Check if the specified session ID exists or not
*
* @param sessionID Session ID to check
* @return true if session exists, false otherwise
*/
boolean sessionExists(String sessionID);
/**
* Get the static info for the given session and worker IDs, or null if no such static info has been reported
*
* @param sessionID Session ID
* @param workerID worker ID
* @return Static info, or null if none has been reported
*/
Persistable getStaticInfo(String sessionID, String typeID, String workerID);
/**
* Get all static informations for the given session and type ID
*
* @param sessionID Session ID to get static info for
* @param typeID Type ID to get static info for
* @return All static info instances matching both the session and type IDs
*/
List<Persistable> getAllStaticInfos(String sessionID, String typeID);
/**
* Get the list of type IDs for the given session ID
*
* @param sessionID Session ID to query
* @return List of type IDs
*/
List<String> listTypeIDsForSession(String sessionID);
/**
* For a given session ID, list all of the known worker IDs
*
* @param sessionID Session ID
* @return List of worker IDs, or possibly null if session ID is unknown
*/
List<String> listWorkerIDsForSession(String sessionID);
/**
* For a given session ID and type ID, list all of the known worker IDs
*
* @param sessionID Session ID
* @param typeID Type ID
* @return List of worker IDs, or possibly null if session ID is unknown
*/
List<String> listWorkerIDsForSessionAndType(String sessionID, String typeID);
/**
* Return the number of update records for the given session ID (all workers)
*
* @param sessionID Session ID
* @return number of update records
*/
int getNumUpdateRecordsFor(String sessionID);
/**
* Return the number of update records for the given session ID and worker ID
*
* @param sessionID Session ID
* @param workerID Worker ID
* @return number of update records
*/
int getNumUpdateRecordsFor(String sessionID, String typeID, String workerID);
/**
* Get the latest update record (i.e., update record with the largest timestamp value) for the specified
* session and worker IDs
*
* @param sessionID session ID
* @param workerID worker ID
* @return UpdateRecord containing the session/worker IDs, timestamp and content for the most recent update
*/
Persistable getLatestUpdate(String sessionID, String typeID, String workerID);
/**
* Get the specified update (or null, if none exists for the given session/worker ids and timestamp)
*
* @param sessionID Session ID
* @param workerID Worker ID
* @param timestamp Timestamp
* @return Update
*/
Persistable getUpdate(String sessionID, String typeId, String workerID, long timestamp);
/**
* Get the latest update for all workers, for the given session ID
*
* @param sessionID Session ID
* @return List of updates for the given Session ID
*/
List<Persistable> getLatestUpdateAllWorkers(String sessionID, String typeID);
/**
* Get all updates for the given session and worker ID, that occur after (not including) the given timestamp.
* Results should be sorted by time.
*
* @param sessionID Session ID
* @param workerID Worker Id
* @param timestamp Timestamp
* @return List of records occurring after the given timestamp
*/
List<Persistable> getAllUpdatesAfter(String sessionID, String typeID, String workerID, long timestamp);
/**
* Get all updates for the given session ID (all worker IDs), that occur after (not including) the given timestamp.
* Results should be sorted by time.
*
* @param sessionID Session ID
* @param timestamp Timestamp
* @return List of records occurring after the given timestamp
*/
List<Persistable> getAllUpdatesAfter(String sessionID, String typeID, long timestamp);
/**
* List the times of all updates for the specified sessionID, typeID and workerID
*
* @param sessionID Session ID to get update times for
* @param typeID Type ID to get update times for
* @param workerID Worker ID to get update times for
* @return Times of all updates
*/
long[] getAllUpdateTimes(String sessionID, String typeID, String workerID);
/**
* Get updates for the specified times only
*
* @param sessionID Session ID to get update times for
* @param typeID Type ID to get update times for
* @param workerID Worker ID to get update times for
* @param timestamps Timestamps to get the updates for. Note that if one of the specified times does not exist,
* it will be ommitted from the returned results list.
* @return List of updates at the specified times
*/
List<Persistable> getUpdates(String sessionID, String typeID, String workerID, long[] timestamps);
/**
* Get the session metadata, if any has been registered via {@link #putStorageMetaData(StorageMetaData)}
*
* @param sessionID Session ID to get metadat
* @return Session metadata, or null if none is available
*/
StorageMetaData getStorageMetaData(String sessionID, String typeID);
// ----- Listeners -----
/**
* Add a new StatsStorageListener. The given listener will called whenever a state change occurs for the stats
* storage instance
*
* @param listener Listener to add
*/
void registerStatsStorageListener(StatsStorageListener listener);
/**
* Remove the specified listener, if it is present.
*
* @param listener Listener to remove
*/
void deregisterStatsStorageListener(StatsStorageListener listener);
/**
* Remove all listeners from the StatsStorage instance
*/
void removeAllListeners();
/**
* Get a list (shallow copy) of all listeners currently present
*
* @return List of listeners
*/
List<StatsStorageListener> getListeners();
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageEvent.java
+35
View File
@@ -0,0 +1,35 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
import lombok.AllArgsConstructor;
import lombok.Data;
@AllArgsConstructor
@Data
public class StatsStorageEvent {
private final StatsStorage statsStorage;
private final StatsStorageListener.EventType eventType;
private final String sessionID;
private final String typeID;
private final String workerID;
private final long timestamp;
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageListener.java
+37
View File
@@ -0,0 +1,37 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
public interface StatsStorageListener {
enum EventType {
NewSessionID, NewTypeID, NewWorkerID, PostMetaData, PostStaticInfo, PostUpdate
}
/**
* Notify will be called whenever an event (new information posted, etc) occurs.
* Processing these events should ideally be done asynchronously.
*
* @param event Event that occurred
*/
void notify(StatsStorageEvent event);
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageRouter.java
+68
View File
@@ -0,0 +1,68 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
import java.util.Collection;
public interface StatsStorageRouter {
/**
* Method to store some additional metadata for each session. Idea: record the classes used to
* serialize and deserialize the static info and updates (as a class name).
* This is mainly used for debugging and validation.
*
* @param storageMetaData Storage metadata to store
*/
void putStorageMetaData(StorageMetaData storageMetaData); //TODO error handling
void putStorageMetaData(Collection<? extends StorageMetaData> storageMetaData);
/**
* Static info: reported once per session, upon initialization
*
* @param staticInfo Static info to store
*/
void putStaticInfo(Persistable staticInfo); //TODO error handling
/**
* Static info: reported once per session, upon initialization
*
* @param staticInfo Static info to store
*/
void putStaticInfo(Collection<? extends Persistable> staticInfo);
/**
* Updates: stored multiple times per session (periodically, for example)
*
* @param update Update info to store
*/
void putUpdate(Persistable update); //TODO error handling
/**
* Updates: stored multiple times per session (periodically, for example)
*
* @param updates Update info to store
*/
void putUpdate(Collection<? extends Persistable> updates);
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageRouterProvider.java
+29
View File
@@ -0,0 +1,29 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
import java.io.Serializable;
public interface StatsStorageRouterProvider extends Serializable {
StatsStorageRouter getRouter();
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StorageMetaData.java
+62
View File
@@ -0,0 +1,62 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
import java.io.Serializable;
public interface StorageMetaData extends Persistable {
/**
* Timestamp for the metadata
*/
long getTimeStamp();
/**
* Session ID for the metadata
*/
String getSessionID();
/**
* Type ID for the metadata
*/
String getTypeID();
/**
* Worker ID for the metadata
*/
String getWorkerID();
/**
* Full class name for the initialization information that will be posted. Is expected to implement {@link Persistable}.
*/
String getInitTypeClass();
/**
* Full class name for the update information that will be posted. Is expected to implement {@link Persistable}.
*/
String getUpdateTypeClass();
/**
* Get extra metadata, if any
*/
Serializable getExtraMetaData();
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StorageType.java
+25
View File
@@ -0,0 +1,25 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage;
public enum StorageType {
MetaData, StaticInfo, Update
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/impl/CollectionStatsStorageRouter.java
+67
View File
@@ -0,0 +1,67 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage.impl;
import lombok.AllArgsConstructor;
import org.deeplearning4j.core.storage.Persistable;
import org.deeplearning4j.core.storage.StatsStorageRouter;
import org.deeplearning4j.core.storage.StorageMetaData;
import java.util.Collection;
@AllArgsConstructor
public class CollectionStatsStorageRouter implements StatsStorageRouter {
private Collection<StorageMetaData> metaDatas;
private Collection<Persistable> staticInfos;
private Collection<Persistable> updates;
@Override
public void putStorageMetaData(StorageMetaData storageMetaData) {
this.metaDatas.add(storageMetaData);
}
@Override
public void putStorageMetaData(Collection<? extends StorageMetaData> storageMetaData) {
this.metaDatas.addAll(storageMetaData);
}
@Override
public void putStaticInfo(Persistable staticInfo) {
this.staticInfos.add(staticInfo);
}
@Override
public void putStaticInfo(Collection<? extends Persistable> staticInfo) {
this.staticInfos.addAll(staticInfo);
}
@Override
public void putUpdate(Persistable update) {
this.updates.add(update);
}
@Override
public void putUpdate(Collection<? extends Persistable> updates) {
this.updates.addAll(updates);
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/impl/RemoteUIStatsStorageRouter.java
+390
View File
@@ -0,0 +1,390 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage.impl;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.binary.Base64;
import org.deeplearning4j.core.storage.Persistable;
import org.deeplearning4j.core.storage.StatsStorageRouter;
import org.deeplearning4j.core.storage.StorageMetaData;
import org.deeplearning4j.core.storage.StorageType;
import org.nd4j.shade.jackson.databind.ObjectMapper;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.*;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@Slf4j
public class RemoteUIStatsStorageRouter implements StatsStorageRouter, Serializable, Closeable {
private static final String ROUTE_IS_DOWN = "Info posted to RemoteUIStatsStorageRouter but router is shut down.";
private static final String MAX_WARNINGS_REACHED = "RemoteUIStatsStorageRouter: Reached max shutdown warnings. No further warnings will be produced.";
/**
* Default path for posting data to the UI - i.e., http://localhost:9000/remoteReceive or similar
*/
public static final String DEFAULT_PATH = "remoteReceive";
/**
* Default maximum number of (consecutive) retries on failure
*/
public static final int DEFAULT_MAX_RETRIES = 10;
/**
* Base delay for retries
*/
public static final long DEFAULT_BASE_RETR_DELAY_MS = 1000;
/**
* Default backoff multiplicative factor for retrying
*/
public static final double DEFAULT_RETRY_BACKOFF_FACTOR = 2.0;
private static final long MAX_SHUTDOWN_WARN_COUNT = 5;
private final String USER_AGENT = "Mozilla/5.0";
private final URL url;
private final int maxRetryCount;
private final long retryDelayMS;
private final double retryBackoffFactor;
private transient LinkedBlockingDeque<ToPost> queue = new LinkedBlockingDeque<>();
private transient Thread postThread;
private AtomicBoolean shutdown = new AtomicBoolean(false);
private AtomicLong shutdownWarnCount = new AtomicLong(0);
private static final ObjectMapper objectMapper = new ObjectMapper();
/**
* Create remote UI with defaults for all values except address
*
* @param address Address of the remote UI: for example, "http://localhost:9000"
*/
public RemoteUIStatsStorageRouter(String address) {
this(address, DEFAULT_MAX_RETRIES, DEFAULT_BASE_RETR_DELAY_MS, DEFAULT_RETRY_BACKOFF_FACTOR);
}
/**
* @param address Address of the remote UI: for example, "http://localhost:9000"
* @param maxRetryCount Maximum number of retries before failing. Set to -1 to always retry
* @param retryDelayMS Base delay before retrying, in milliseconds
* @param retryBackoffFactor Backoff factor for retrying: 2.0 for example gives delays of 1000, 2000, 4000, 8000,
* etc milliseconds, with a base retry delay of 1000
*/
public RemoteUIStatsStorageRouter(String address, int maxRetryCount, long retryDelayMS, double retryBackoffFactor) {
this(address, DEFAULT_PATH, maxRetryCount, retryDelayMS, retryBackoffFactor);
}
/**
* @param address Address of the remote UI: for example, "http://localhost:9000"
* @param path Path/endpoint to post to: for example "remoteReceive" -> added to path to become like
* "http://localhost:9000/remoteReceive"
* @param maxRetryCount Maximum number of retries before failing. Set to -1 to always retry
* @param retryDelayMS Base delay before retrying, in milliseconds
* @param retryBackoffFactor Backoff factor for retrying: 2.0 for example gives delays of 1000, 2000, 4000, 8000,
* etc milliseconds, with a base retry delay of 1000
*/
public RemoteUIStatsStorageRouter(String address, String path, int maxRetryCount, long retryDelayMS,
double retryBackoffFactor) {
this.maxRetryCount = maxRetryCount;
this.retryDelayMS = retryDelayMS;
this.retryBackoffFactor = retryBackoffFactor;
String url = address;
if (path != null) {
if (url.endsWith("/")) {
url = url + path;
} else {
url = url + "/" + path;
}
}
try {
this.url = new URL(url);
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}
}
@Override
public void close(){
shutdown();
}
public void shutdown(){
this.shutdown.set(true);
}
private synchronized void checkThread(){
if(postThread == null){
postThread = new Thread(new PostRunnable());
postThread.setDaemon(true);
postThread.start();
}
if(queue == null){
//May be null if router has been deserialized
queue = new LinkedBlockingDeque<>();
}
}
@Override
public void putStorageMetaData(StorageMetaData storageMetaData) {
putStorageMetaData(Collections.singleton(storageMetaData));
}
@Override
public void putStorageMetaData(Collection<? extends StorageMetaData> storageMetaData) {
checkThread();
if (shutdown.get()) {
long count = shutdownWarnCount.getAndIncrement();
if (count <= MAX_SHUTDOWN_WARN_COUNT) {
log.warn(ROUTE_IS_DOWN);
}
if (count == MAX_SHUTDOWN_WARN_COUNT) {
log.warn(MAX_WARNINGS_REACHED);
}
} else {
for (StorageMetaData m : storageMetaData) {
queue.add(new ToPost(m, null, null));
}
}
}
@Override
public void putStaticInfo(Persistable staticInfo) {
putStaticInfo(Collections.singletonList(staticInfo));
}
@Override
public void putStaticInfo(Collection<? extends Persistable> staticInfo) {
checkThread();
if (shutdown.get()) {
long count = shutdownWarnCount.getAndIncrement();
if (count <= MAX_SHUTDOWN_WARN_COUNT) {
log.warn(ROUTE_IS_DOWN);
}
if (count == MAX_SHUTDOWN_WARN_COUNT) {
log.warn(MAX_WARNINGS_REACHED);
}
} else {
for (Persistable p : staticInfo) {
queue.add(new ToPost(null, p, null));
}
}
}
@Override
public void putUpdate(Persistable update) {
putUpdate(Collections.singleton(update));
}
@Override
public void putUpdate(Collection<? extends Persistable> updates) {
checkThread();
if (shutdown.get()) {
long count = shutdownWarnCount.getAndIncrement();
if (count <= MAX_SHUTDOWN_WARN_COUNT) {
log.warn(ROUTE_IS_DOWN);
}
if (count == MAX_SHUTDOWN_WARN_COUNT) {
log.warn(MAX_WARNINGS_REACHED);
}
} else {
for (Persistable p : updates) {
queue.add(new ToPost(null, null, p));
}
}
}
@AllArgsConstructor
@Data
private static class ToPost {
private final StorageMetaData meta;
private final Persistable staticInfo;
private final Persistable update;
}
//Runnable class for doing async posting
private class PostRunnable implements Runnable {
private int failureCount = 0;
private long nextDelayMs = retryDelayMS;
@Override
public void run() {
try {
runHelper();
} catch (Exception e) {
log.error("Exception encountered in remote UI posting thread. Shutting down.", e);
shutdown.set(true);
}
}
private void runHelper() {
while (!shutdown.get()) {
List<ToPost> list = new ArrayList<>();
ToPost t;
try {
t = queue.take(); //Blocking operation
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
continue;
}
list.add(t);
queue.drainTo(list); //Non-blocking
int successCount = 0;
for (ToPost toPost : list) {
boolean success;
try {
success = tryPost(toPost);
} catch (IOException e) {
failureCount++;
log.warn("Error posting to remote UI at {}, consecutive failure count = {}. Waiting {} ms before retrying",
url, failureCount, nextDelayMs, e);
success = false;
}
if (!success) {
for (int i = list.size() - 1; i > successCount; i--) {
queue.addFirst(list.get(i)); //Add remaining back to be processed in original order
}
waitForRetry();
break;
} else {
successCount++;
failureCount = 0;
nextDelayMs = retryDelayMS;
}
}
}
}
private void waitForRetry() {
if (maxRetryCount >= 0 && failureCount > maxRetryCount) {
throw new RuntimeException("RemoteUIStatsStorageRouter: hit maximum consecutive failures("
+ maxRetryCount + "). Shutting down remote router thread");
} else {
try {
Thread.sleep(nextDelayMs);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
nextDelayMs *= retryBackoffFactor;
}
}
}
private HttpURLConnection getConnection() throws IOException {
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("POST");
connection.setRequestProperty("User-Agent", USER_AGENT);
connection.setRequestProperty("Content-Type", "application/json");
connection.setDoOutput(true);
return connection;
}
private boolean tryPost(ToPost toPost) throws IOException {
HttpURLConnection connection = getConnection();
String className;
byte[] asBytes;
StorageType type;
if (toPost.getMeta() != null) {
StorageMetaData smd = toPost.getMeta();
className = smd.getClass().getName();
asBytes = smd.encode();
type = StorageType.MetaData;
} else if (toPost.getStaticInfo() != null) {
Persistable p = toPost.getStaticInfo();
className = p.getClass().getName();
asBytes = p.encode();
type = StorageType.StaticInfo;
} else {
Persistable p = toPost.getUpdate();
className = p.getClass().getName();
asBytes = p.encode();
type = StorageType.Update;
}
String base64 = Base64.encodeBase64String(asBytes);
Map<String, String> jsonObj = new LinkedHashMap<>();
jsonObj.put("type", type.name());
jsonObj.put("class", className);
jsonObj.put("data", base64);
String str;
try {
str = objectMapper.writeValueAsString(jsonObj);
} catch (Exception e) {
throw new RuntimeException(e); //Should never get an exception from simple Map<String,String>
}
DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
dos.writeBytes(str);
dos.flush();
dos.close();
try {
int responseCode = connection.getResponseCode();
if (responseCode != 200) {
BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
String inputLine;
StringBuilder response = new StringBuilder();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
log.warn("Error posting to remote UI - received response code {}\tContent: {}", response,
response.toString());
return false;
}
} catch (IOException e) {
String msg = e.getMessage();
if (msg.contains("403 for URL")) {
log.warn("Error posting to remote UI at {} (Response code: 403)."
+ " Remote listener support is not enabled? use UIServer.getInstance().enableRemoteListener()",
url, e);
} else {
log.warn("Error posting to remote UI at {}", url, e);
}
return false;
}
return true;
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/listener/RoutingIterationListener.java
+45
View File
@@ -0,0 +1,45 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.storage.listener;
import org.deeplearning4j.core.storage.StatsStorageRouter;
import org.deeplearning4j.core.storage.StatsStorageRouterProvider;
import org.deeplearning4j.optimize.api.TrainingListener;
import java.io.Serializable;
public interface RoutingIterationListener extends TrainingListener, Cloneable, Serializable {
void setStorageRouter(StatsStorageRouter router);
StatsStorageRouter getStorageRouter();
void setWorkerID(String workerID);
String getWorkerID();
void setSessionID(String sessionID);
String getSessionID();
RoutingIterationListener clone();
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/ui/UiConnectionInfo.java
+150
View File
@@ -0,0 +1,150 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.ui;
import lombok.Data;
import lombok.NonNull;
@Data
public class UiConnectionInfo {
private String sessionId;
private String login;
private String password;
private String address = "localhost";
private int port = 8080;
private String path = "";
private boolean useHttps;
public UiConnectionInfo() {
this.sessionId = java.util.UUID.randomUUID().toString();
}
public void setSessionId(@NonNull String sessionId) {
this.sessionId = sessionId;
}
/**
* This method returns scheme, address and port for this UiConnectionInfo
*
* i.e: https://localhost:8080
*
* @return
*/
public String getFirstPart() {
StringBuilder builder = new StringBuilder();
builder.append(useHttps ? "https" : "http").append("://").append(address).append(":").append(port).append("");
return builder.toString();
}
public String getSecondPart() {
return getSecondPart("");
}
public String getSecondPart(String nPath) {
StringBuilder builder = new StringBuilder();
if (path != null && !path.isEmpty()) {
builder.append(path.startsWith("/") ? path : ("/" + path)).append("/");
}
if (nPath != null) {
nPath = nPath.replaceFirst("^/", "");
builder.append(nPath.startsWith("/") ? nPath : ("/" + nPath)).append("/");
}
return builder.toString().replaceAll("\\/{2,}", "/");
}
public String getFullAddress(String nPath) {
if (nPath == null || nPath.isEmpty()) {
return getFullAddress();
} else {
return getFirstPart() + getSecondPart(nPath) + "?sid=" + this.getSessionId();
}
}
public String getFullAddress() {
return getFirstPart() + getSecondPart();
}
public static class Builder {
private UiConnectionInfo info = new UiConnectionInfo();
/**
* This method allows you to specify sessionId for this UiConnectionInfo instance
*
* PLEASE NOTE: This is not recommended. Advised behaviour - keep it random, as is.
*
* @param sessionId
* @return
*/
public Builder setSessionId(@NonNull String sessionId) {
info.setSessionId(sessionId);
return this;
}
public Builder setLogin(@NonNull String login) {
info.setLogin(login);
return this;
}
public Builder setPassword(String password) {
info.setPassword(password);
return this;
}
public Builder setAddress(@NonNull String address) {
info.setAddress(address);
return this;
}
public Builder setPort(int port) {
if (port <= 0)
throw new IllegalStateException("UiServer port can't be <= 0");
info.setPort(port);
return this;
}
public Builder enableHttps(boolean reallyEnable) {
info.setUseHttps(reallyEnable);
return this;
}
/**
* If you're using UiServer as servlet, located not at root folder of webserver (i.e. http://yourdomain.com/somepath/webui/), you can set path here.
* For provided example path will be "/somepath/webui/"
*
* @param path
* @return
*/
public Builder setPath(String path) {
info.setPath(path);
return this;
}
public UiConnectionInfo build() {
return info;
}
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/ModelGuesser.java
+222
View File
@@ -0,0 +1,222 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.util;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.deeplearning4j.common.util.ND4JFileUtils;
import org.deeplearning4j.config.DL4JSystemProperties;
import org.deeplearning4j.nn.api.Model;
import org.deeplearning4j.nn.conf.ComputationGraphConfiguration;
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.modelimport.keras.KerasModelImport;
import org.deeplearning4j.util.ModelSerializer;
import org.nd4j.linalg.dataset.api.preprocessor.Normalizer;
import java.io.*;
import java.util.UUID;
/**
* Guess a model from the given path
* @author Adam Gibson
*/
@Slf4j
public class ModelGuesser {
/**
* A facade for {@link ModelSerializer#restoreNormalizerFromInputStream(InputStream)}
* @param is the input stream to load form
* @return the loaded normalizer
* @throws IOException
*/
public static Normalizer<?> loadNormalizer(InputStream is) throws IOException {
return ModelSerializer.restoreNormalizerFromInputStream(is);
}
/**
* A facade for {@link ModelSerializer#restoreNormalizerFromFile(File)}
* @param path the path to the file
* @return the loaded normalizer
*/
public static Normalizer<?> loadNormalizer(String path) {
try {
return ModelSerializer.restoreNormalizerFromFile(new File(path));
} catch (IOException e){
throw new RuntimeException(e);
}
}
/**
* Load the model from the given file path
* @param path the path of the file to "guess"
*
* @return the loaded model
* @throws Exception
*/
public static Object loadConfigGuess(String path) throws Exception {
String input = FileUtils.readFileToString(new File(path));
//note here that we load json BEFORE YAML. YAML
//turns out to load just fine *accidentally*
try {
return MultiLayerConfiguration.fromJson(input);
} catch (Exception e) {
log.warn("Tried multi layer config from json", e);
try {
return KerasModelImport.importKerasModelConfiguration(path);
} catch (Exception e1) {
log.warn("Tried keras model config", e);
try {
return KerasModelImport.importKerasSequentialConfiguration(path);
} catch (Exception e2) {
log.warn("Tried keras sequence config", e);
try {
return ComputationGraphConfiguration.fromJson(input);
} catch (Exception e3) {
log.warn("Tried computation graph from json");
try {
return MultiLayerConfiguration.fromYaml(input);
} catch (Exception e4) {
log.warn("Tried multi layer configuration from yaml");
try {
return ComputationGraphConfiguration.fromYaml(input);
} catch (Exception e5) {
throw new ModelGuesserException("Unable to load configuration from path " + path
+ " (invalid config file or not a known config type)");
}
}
}
}
}
}
}
/**
* Load the model from the given input stream
* @param stream the path of the file to "guess"
*
* @return the loaded model
* @throws Exception
*/
public static Object loadConfigGuess(InputStream stream) throws Exception {
String p = System.getProperty(DL4JSystemProperties.DL4J_TEMP_DIR_PROPERTY);
File tmp = ND4JFileUtils.createTempFile("model-" + UUID.randomUUID().toString(), "bin");
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(tmp));
IOUtils.copy(stream, bufferedOutputStream);
bufferedOutputStream.flush();
bufferedOutputStream.close();
tmp.deleteOnExit();
try {
return loadConfigGuess(tmp.getAbsolutePath());
} finally {
tmp.delete();
}
}
/**
* Load the model from the given file path
* @param path the path of the file to "guess"
*
* @return the loaded model
* @throws Exception
*/
public static Model loadModelGuess(String path) throws Exception {
try {
return ModelSerializer.restoreMultiLayerNetwork(new File(path), true);
} catch (Exception e) {
log.warn("Tried multi layer network");
try {
return ModelSerializer.restoreComputationGraph(new File(path), true);
} catch (Exception e1) {
log.warn("Tried computation graph");
try {
return ModelSerializer.restoreMultiLayerNetwork(new File(path), false);
} catch (Exception e4) {
try {
return ModelSerializer.restoreComputationGraph(new File(path), false);
} catch (Exception e5) {
try {
return KerasModelImport.importKerasModelAndWeights(path);
} catch (Exception e2) {
log.warn("Tried multi layer network keras");
try {
return KerasModelImport.importKerasSequentialModelAndWeights(path);
} catch (Exception e3) {
throw new ModelGuesserException("Unable to load model from path " + path
+ " (invalid model file or not a known model type)");
}
}
}
}
}
}
}
/**
* Load the model from the given input stream. The content of the stream is written to a temporary location,
* see {@link DL4JSystemProperties#DL4J_TEMP_DIR_PROPERTY}
*
* @param stream the path of the file to "guess"
*
* @return the loaded model
* @throws Exception
*/
public static Model loadModelGuess(InputStream stream) throws Exception {
return loadModelGuess(stream, null);
}
/**
* As per {@link #loadModelGuess(InputStream)} but (optionally) allows copying to the specified temporary directory
* @param stream Stream of the model file
* @param tempDirectory Temporary/working directory. May be null.
*/
public static Model loadModelGuess(InputStream stream, File tempDirectory) throws Exception {
//Currently (Nov 2017): KerasModelImport doesn't support loading from input streams
//Simplest solution here: write to a temporary file
File f;
if(tempDirectory == null){
f = ND4JFileUtils.createTempFile("loadModelGuess",".bin");
} else {
f = File.createTempFile("loadModelGuess", ".bin", tempDirectory);
}
f.deleteOnExit();
try (OutputStream os = new BufferedOutputStream(new FileOutputStream(f))) {
IOUtils.copy(stream, os);
os.flush();
return loadModelGuess(f.getAbsolutePath());
} catch (ModelGuesserException e){
throw new ModelGuesserException("Unable to load model from input stream (invalid model file not a known model type)");
} finally {
f.delete();
}
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/ModelGuesserException.java
+36
View File
@@ -0,0 +1,36 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.util;
public class ModelGuesserException extends Exception {
public ModelGuesserException(String message) {
super(message);
}
public ModelGuesserException(String message, Throwable cause) {
super(message, cause);
}
public ModelGuesserException(Throwable cause) {
super(cause);
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/MovingWindowMatrix.java
+124
View File
@@ -0,0 +1,124 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.util;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.factory.Nd4j;
import java.util.ArrayList;
import java.util.List;
/**
*
* Moving window on a matrix (usually used for images)
*
* Given a: This is a list of flattened arrays:
* 1 1 1 1 1 1 2 2
* 2 2 2 2 ----> 1 1 2 2
* 3 3 3 3 3 3 4 4
* 4 4 4 4 3 3 4 4
*
* @author Adam Gibson
*/
public class MovingWindowMatrix {
private int windowRowSize = 28;
private int windowColumnSize = 28;
private INDArray toSlice;
private boolean addRotate = false;
/**
*
* @param toSlice matrix to slice
* @param windowRowSize the number of rows in each window
* @param windowColumnSize the number of columns in each window
* @param addRotate whether to add the possible rotations of each moving window
*/
public MovingWindowMatrix(INDArray toSlice, int windowRowSize, int windowColumnSize, boolean addRotate) {
this.toSlice = toSlice;
this.windowRowSize = windowRowSize;
this.windowColumnSize = windowColumnSize;
this.addRotate = addRotate;
}
/**
* Same as calling new MovingWindowMatrix(toSlice,windowRowSize,windowColumnSize,false)
* @param toSlice
* @param windowRowSize
* @param windowColumnSize
*/
public MovingWindowMatrix(INDArray toSlice, int windowRowSize, int windowColumnSize) {
this(toSlice, windowRowSize, windowColumnSize, false);
}
/**
* Returns a list of non flattened moving window matrices
* @return the list of matrices
*/
public List<INDArray> windows() {
return windows(false);
}
/**
* Moving window, capture a row x column moving window of
* a given matrix
* @param flattened whether the arrays should be flattened or not
* @return the list of moving windows
*/
public List<INDArray> windows(boolean flattened) {
List<INDArray> ret = new ArrayList<>();
int window = 0;
for (int i = 0; i < toSlice.length(); i++) {
if (window >= toSlice.length())
break;
double[] w = new double[this.windowRowSize * this.windowColumnSize];
for (int count = 0; count < this.windowRowSize * this.windowColumnSize; count++) {
w[count] = toSlice.getDouble(count + window);
}
INDArray add = Nd4j.create(w);
if (flattened)
add = add.ravel();
else
add = add.reshape(windowRowSize, windowColumnSize);
if (addRotate) {
INDArray currRotation = add.dup();
//3 different orientations besides the original
for (int rotation = 0; rotation < 3; rotation++) {
Nd4j.rot90(currRotation);
ret.add(currRotation.dup());
}
}
window += this.windowRowSize * this.windowColumnSize;
ret.add(add);
}
return ret;
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/ThreadUtils.java
+51
View File
@@ -0,0 +1,51 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.util;
import java.util.concurrent.locks.LockSupport;
public class ThreadUtils {
public static void uncheckedSleep(long millis) {
LockSupport.parkNanos(millis * 1000000);
// we must check the interrupted status in case this is used in a loop
// Otherwise we may end up spinning 100% without breaking out on an interruption
if (Thread.currentThread().isInterrupted()) {
throw new UncheckedInterruptedException();
}
}
public static void uncheckedSleepNanos(long nanos) {
LockSupport.parkNanos(nanos);
// we must check the interrupted status in case this is used in a loop
// Otherwise we may end up spinning 100% without breaking out on an interruption
if (Thread.currentThread().isInterrupted()) {
throw new UncheckedInterruptedException();
}
}
/**
* Similar to {@link InterruptedException} in concept, but unchecked. Allowing this to be thrown without being
* explicitly declared in the API.
*/
public static class UncheckedInterruptedException extends RuntimeException {
}
}
deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/UIDProvider.java
+104
View File
@@ -0,0 +1,104 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.core.util;
import lombok.extern.slf4j.Slf4j;
import java.net.NetworkInterface;
import java.rmi.server.UID;
import java.util.Enumeration;
@Slf4j
public class UIDProvider {
private static final String JVM_UID;
private static final String HARDWARE_UID;
static {
UID jvmUIDSource = new UID();
String asString = jvmUIDSource.toString();
//Format here: hexStringFromRandomNumber:hexStringFromSystemClock:hexStringOfUIDInstance
//The first two components here will be identical for all UID instances in a JVM, where as the 'hexStringOfUIDInstance'
// will vary (increment) between UID object instances. So we'll only be using the first two components here
int lastIdx = asString.lastIndexOf(":");
JVM_UID = asString.substring(0, lastIdx).replaceAll(":", "");
//Assumptions here:
//1. getNetworkInterfaces() returns at least one non-null element
// This is guaranteed by getNetworkInterfaces() Javadoc: "The {@code Enumeration} contains at least one element..."
//2. That the iteration order for network interfaces is consistent between JVM instances on the same hardware
// This appears to hold, but no formal guarantees seem to be available here
//3. That MAC addresses are 'unique enough' for our purposes
byte[] address = null;
boolean noInterfaces = false;
Enumeration<NetworkInterface> niEnumeration = null;
try {
niEnumeration = NetworkInterface.getNetworkInterfaces();
} catch (Exception e) {
noInterfaces = true;
}
if (niEnumeration != null) {
while (niEnumeration.hasMoreElements()) {
NetworkInterface ni = niEnumeration.nextElement();
byte[] addr;
try {
addr = ni.getHardwareAddress();
} catch (Exception e) {
continue;
}
if (addr == null || addr.length != 6)
continue; //May be null (if it can't be obtained) or not standard 6 byte MAC-48 representation
address = addr;
break;
}
}
if (address == null) {
log.warn("Could not generate hardware UID{}. Using fallback: JVM UID as hardware UID.",
(noInterfaces ? " (no interfaces)" : ""));
HARDWARE_UID = JVM_UID;
} else {
StringBuilder sb = new StringBuilder();
for (byte b : address) {
sb.append(String.format("%02x", b));
}
HARDWARE_UID = sb.toString();
}
}
private UIDProvider() {}
public static String getJVMUID() {
return JVM_UID;
}
public static String getHardwareUID() {
return HARDWARE_UID;
}
}
deeplearning4j/deeplearning4j-core/src/main/java9/module-info.java
+30
View File
@@ -0,0 +1,30 @@
open module deeplearning4j.core {
requires commons.codec;
requires commons.io;
requires deeplearning4j.datavec.iterators;
requires deeplearning4j.modelimport;
requires deeplearning4j.ui.components;
requires jackson;
requires java.desktop;
requires java.rmi;
requires oshi.core;
requires resources;
requires slf4j.api;
requires datavec.api;
requires deeplearning4j.nn;
requires nd4j.api;
requires nd4j.common;
requires oshi.json;
exports org.deeplearning4j.core.datasets.test;
exports org.deeplearning4j.core.datasets.vectorizer;
exports org.deeplearning4j.core.evaluation;
exports org.deeplearning4j.core.listener;
exports org.deeplearning4j.core.loader;
exports org.deeplearning4j.core.loader.impl;
exports org.deeplearning4j.core.parallelism;
exports org.deeplearning4j.core.storage;
exports org.deeplearning4j.core.storage.impl;
exports org.deeplearning4j.core.storage.listener;
exports org.deeplearning4j.core.ui;
exports org.deeplearning4j.core.util;
}
deeplearning4j/deeplearning4j-core/src/main/resources/iris.dat
Executable
+150
View File
@@ -0,0 +1,150 @@
5.1,3.5,1.4,0.2,0
4.9,3.0,1.4,0.2,0
4.7,3.2,1.3,0.2,0
4.6,3.1,1.5,0.2,0
5.0,3.6,1.4,0.2,0
5.4,3.9,1.7,0.4,0
4.6,3.4,1.4,0.3,0
5.0,3.4,1.5,0.2,0
4.4,2.9,1.4,0.2,0
4.9,3.1,1.5,0.1,0
5.4,3.7,1.5,0.2,0
4.8,3.4,1.6,0.2,0
4.8,3.0,1.4,0.1,0
4.3,3.0,1.1,0.1,0
5.8,4.0,1.2,0.2,0
5.7,4.4,1.5,0.4,0
5.4,3.9,1.3,0.4,0
5.1,3.5,1.4,0.3,0
5.7,3.8,1.7,0.3,0
5.1,3.8,1.5,0.3,0
5.4,3.4,1.7,0.2,0
5.1,3.7,1.5,0.4,0
4.6,3.6,1.0,0.2,0
5.1,3.3,1.7,0.5,0
4.8,3.4,1.9,0.2,0
5.0,3.0,1.6,0.2,0
5.0,3.4,1.6,0.4,0
5.2,3.5,1.5,0.2,0
5.2,3.4,1.4,0.2,0
4.7,3.2,1.6,0.2,0
4.8,3.1,1.6,0.2,0
5.4,3.4,1.5,0.4,0
5.2,4.1,1.5,0.1,0
5.5,4.2,1.4,0.2,0
4.9,3.1,1.5,0.1,0
5.0,3.2,1.2,0.2,0
5.5,3.5,1.3,0.2,0
4.9,3.1,1.5,0.1,0
4.4,3.0,1.3,0.2,0
5.1,3.4,1.5,0.2,0
5.0,3.5,1.3,0.3,0
4.5,2.3,1.3,0.3,0
4.4,3.2,1.3,0.2,0
5.0,3.5,1.6,0.6,0
5.1,3.8,1.9,0.4,0
4.8,3.0,1.4,0.3,0
5.1,3.8,1.6,0.2,0
4.6,3.2,1.4,0.2,0
5.3,3.7,1.5,0.2,0
5.0,3.3,1.4,0.2,0
7.0,3.2,4.7,1.4,1
6.4,3.2,4.5,1.5,1
6.9,3.1,4.9,1.5,1
5.5,2.3,4.0,1.3,1
6.5,2.8,4.6,1.5,1
5.7,2.8,4.5,1.3,1
6.3,3.3,4.7,1.6,1
4.9,2.4,3.3,1.0,1
6.6,2.9,4.6,1.3,1
5.2,2.7,3.9,1.4,1
5.0,2.0,3.5,1.0,1
5.9,3.0,4.2,1.5,1
6.0,2.2,4.0,1.0,1
6.1,2.9,4.7,1.4,1
5.6,2.9,3.6,1.3,1
6.7,3.1,4.4,1.4,1
5.6,3.0,4.5,1.5,1
5.8,2.7,4.1,1.0,1
6.2,2.2,4.5,1.5,1
5.6,2.5,3.9,1.1,1
5.9,3.2,4.8,1.8,1
6.1,2.8,4.0,1.3,1
6.3,2.5,4.9,1.5,1
6.1,2.8,4.7,1.2,1
6.4,2.9,4.3,1.3,1
6.6,3.0,4.4,1.4,1
6.8,2.8,4.8,1.4,1
6.7,3.0,5.0,1.7,1
6.0,2.9,4.5,1.5,1
5.7,2.6,3.5,1.0,1
5.5,2.4,3.8,1.1,1
5.5,2.4,3.7,1.0,1
5.8,2.7,3.9,1.2,1
6.0,2.7,5.1,1.6,1
5.4,3.0,4.5,1.5,1
6.0,3.4,4.5,1.6,1
6.7,3.1,4.7,1.5,1
6.3,2.3,4.4,1.3,1
5.6,3.0,4.1,1.3,1
5.5,2.5,4.0,1.3,1
5.5,2.6,4.4,1.2,1
6.1,3.0,4.6,1.4,1
5.8,2.6,4.0,1.2,1
5.0,2.3,3.3,1.0,1
5.6,2.7,4.2,1.3,1
5.7,3.0,4.2,1.2,1
5.7,2.9,4.2,1.3,1
6.2,2.9,4.3,1.3,1
5.1,2.5,3.0,1.1,1
5.7,2.8,4.1,1.3,1
6.3,3.3,6.0,2.5,2
5.8,2.7,5.1,1.9,2
7.1,3.0,5.9,2.1,2
6.3,2.9,5.6,1.8,2
6.5,3.0,5.8,2.2,2
7.6,3.0,6.6,2.1,2
4.9,2.5,4.5,1.7,2
7.3,2.9,6.3,1.8,2
6.7,2.5,5.8,1.8,2
7.2,3.6,6.1,2.5,2
6.5,3.2,5.1,2.0,2
6.4,2.7,5.3,1.9,2
6.8,3.0,5.5,2.1,2
5.7,2.5,5.0,2.0,2
5.8,2.8,5.1,2.4,2
6.4,3.2,5.3,2.3,2
6.5,3.0,5.5,1.8,2
7.7,3.8,6.7,2.2,2
7.7,2.6,6.9,2.3,2
6.0,2.2,5.0,1.5,2
6.9,3.2,5.7,2.3,2
5.6,2.8,4.9,2.0,2
7.7,2.8,6.7,2.0,2
6.3,2.7,4.9,1.8,2
6.7,3.3,5.7,2.1,2
7.2,3.2,6.0,1.8,2
6.2,2.8,4.8,1.8,2
6.1,3.0,4.9,1.8,2
6.4,2.8,5.6,2.1,2
7.2,3.0,5.8,1.6,2
7.4,2.8,6.1,1.9,2
7.9,3.8,6.4,2.0,2
6.4,2.8,5.6,2.2,2
6.3,2.8,5.1,1.5,2
6.1,2.6,5.6,1.4,2
7.7,3.0,6.1,2.3,2
6.3,3.4,5.6,2.4,2
6.4,3.1,5.5,1.8,2
6.0,3.0,4.8,1.8,2
6.9,3.1,5.4,2.1,2
6.7,3.1,5.6,2.4,2
6.9,3.1,5.1,2.3,2
5.8,2.7,5.1,1.9,2
6.8,3.2,5.9,2.3,2
6.7,3.3,5.7,2.5,2
6.7,3.0,5.2,2.3,2
6.3,2.5,5.0,1.9,2
6.5,3.0,5.2,2.0,2
6.2,3.4,5.4,2.3,2
5.9,3.0,5.1,1.8,2