chore: import upstream snapshot with attribution

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/activation/PathUpdate.java

@@ -0,0 +1,35 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.activation;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.io.Serializable;
+
+/**
+ * @author Adam Gibson
+ */
+public @Data @NoArgsConstructor class PathUpdate implements Serializable {
+    private String path;
+
+
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/nearestneighbors/word2vec/NearestNeighborsQuery.java

@@ -0,0 +1,76 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.nearestneighbors.word2vec;
+
+import java.io.Serializable;
+
+/**
+ * @author Adam Gibson
+ */
+public class NearestNeighborsQuery implements Serializable {
+    private String word;
+    private int numWords;
+
+    public NearestNeighborsQuery(String word, int numWords) {
+        this.word = word;
+        this.numWords = numWords;
+    }
+
+    public NearestNeighborsQuery() {}
+
+    public String getWord() {
+        return word;
+    }
+
+    public void setWord(String word) {
+        this.word = word;
+    }
+
+    public int getNumWords() {
+        return numWords;
+    }
+
+    public void setNumWords(int numWords) {
+        this.numWords = numWords;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        NearestNeighborsQuery that = (NearestNeighborsQuery) o;
+
+        if (numWords != that.numWords)
+            return false;
+        return !(word != null ? !word.equals(that.word) : that.word != null);
+
+    }
+
+    @Override
+    public int hashCode() {
+        int result = word != null ? word.hashCode() : 0;
+        result = 31 * result + numWords;
+        return result;
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/renders/PathUpdate.java

@@ -0,0 +1,35 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.renders;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.io.Serializable;
+
+/**
+ * @author Adam Gibson
+ */
+public @Data @NoArgsConstructor class PathUpdate implements Serializable {
+    private String path;
+
+
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/BaseStatsListener.java

@@ -0,0 +1,789 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
+import org.bytedeco.javacpp.Pointer;
+import org.deeplearning4j.config.DL4JClassLoading;
+import org.deeplearning4j.core.storage.StatsStorageRouter;
+import org.deeplearning4j.core.storage.StorageMetaData;
+import org.deeplearning4j.core.storage.listener.RoutingIterationListener;
+import org.deeplearning4j.nn.api.Layer;
+import org.deeplearning4j.nn.api.Model;
+import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
+import org.deeplearning4j.nn.gradient.Gradient;
+import org.deeplearning4j.nn.graph.ComputationGraph;
+import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
+import org.deeplearning4j.ui.model.stats.api.*;
+import org.deeplearning4j.ui.model.storage.FileStatsStorage;
+import org.deeplearning4j.ui.model.storage.InMemoryStatsStorage;
+import org.deeplearning4j.ui.model.stats.impl.DefaultStatsInitializationConfiguration;
+import org.deeplearning4j.ui.model.stats.impl.DefaultStatsUpdateConfiguration;
+import org.deeplearning4j.core.util.UIDProvider;
+import org.nd4j.linalg.api.buffer.util.DataTypeUtil;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.common.primitives.Pair;
+import org.nd4j.nativeblas.NativeOps;
+import org.nd4j.nativeblas.NativeOpsHolder;
+
+import java.io.InputStream;
+import java.io.Serializable;
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
+import java.lang.management.RuntimeMXBean;
+import java.util.*;
+
+@Slf4j
+public abstract class BaseStatsListener implements RoutingIterationListener {
+    public static final String TYPE_ID = "StatsListener";
+
+    private enum StatType {
+        Mean, Stdev, MeanMagnitude
+    }
+
+    private StatsStorageRouter router;
+    private final StatsInitializationConfiguration initConfig;
+    private StatsUpdateConfiguration updateConfig;
+    private String sessionID;
+    private String workerID;
+
+    private transient List<GarbageCollectorMXBean> gcBeans;
+    private Map<String, Pair<Long, Long>> gcStatsAtLastReport;
+
+    //NOTE: may have multiple models, due to multiple pretrain layers all using the same StatsListener
+    private List<ModelInfo> modelInfos = new ArrayList<>();
+
+    private Map<String, Histogram> activationHistograms;
+    private Map<String, Double> meanActivations;        //TODO replace with Eclipse collections primitive maps...
+    private Map<String, Double> stdevActivations;
+    private Map<String, Double> meanMagActivations;
+
+    private Map<String, Histogram> gradientHistograms;
+    private Map<String, Double> meanGradients;        //TODO replace with Eclipse collections primitive maps...
+    private Map<String, Double> stdevGradient;
+    private Map<String, Double> meanMagGradients;
+
+    private static class ModelInfo implements Serializable {
+        private final Model model;
+        private long initTime;
+        private long lastReportTime = -1;
+        private int lastReportIteration = -1;
+        private int examplesSinceLastReport = 0;
+        private int minibatchesSinceLastReport = 0;
+
+        private long totalExamples = 0;
+        private long totalMinibatches = 0;
+
+        private int iterCount = 0;
+
+        private ModelInfo(Model model) {
+            this.model = model;
+        }
+    }
+
+    private ModelInfo getModelInfo(Model model) {
+        ModelInfo mi = null;
+        for (ModelInfo m : modelInfos) {
+            if (m.model == model) {
+                mi = m;
+                break;
+            }
+        }
+        if (mi == null) {
+            mi = new ModelInfo(model);
+            modelInfos.add(mi);
+        }
+        return mi;
+    }
+
+    /**
+     * Create a StatsListener with network information collected at every iteration.
+     *
+     * @param router Where/how to store the calculated stats. For example, {@link InMemoryStatsStorage} or
+     *               {@link FileStatsStorage}
+     */
+    public BaseStatsListener(StatsStorageRouter router) {
+        this(router, null, null, null, null);
+    }
+
+    /**
+     * Create a StatsListener with network information collected every n >= 1 time steps
+     *
+     * @param router            Where/how to store the calculated stats. For example, {@link InMemoryStatsStorage} or
+     *                          {@link FileStatsStorage}
+     * @param listenerFrequency Frequency with which to collect stats information
+     */
+    public BaseStatsListener(StatsStorageRouter router, int listenerFrequency) {
+        this(router, null, new DefaultStatsUpdateConfiguration.Builder().reportingFrequency(listenerFrequency).build(),
+                null, null);
+    }
+
+    public BaseStatsListener(StatsStorageRouter router, StatsInitializationConfiguration initConfig,
+                             StatsUpdateConfiguration updateConfig, String sessionID, String workerID) {
+        this.router = router;
+        if (initConfig == null) {
+            this.initConfig = new DefaultStatsInitializationConfiguration(true, true, true);
+        } else {
+            this.initConfig = initConfig;
+        }
+        if (updateConfig == null) {
+            this.updateConfig = new DefaultStatsUpdateConfiguration.Builder().build();
+        } else {
+            this.updateConfig = updateConfig;
+        }
+        if (sessionID == null) {
+            //TODO handle syncing session IDs across different listeners in the same model...
+            this.sessionID = UUID.randomUUID().toString();
+        } else {
+            this.sessionID = sessionID;
+        }
+        if (workerID == null) {
+            this.workerID = UIDProvider.getJVMUID() + "_" + Thread.currentThread().getId();
+        } else {
+            this.workerID = workerID;
+        }
+    }
+
+    public abstract StatsInitializationReport getNewInitializationReport();
+
+    public abstract StatsReport getNewStatsReport();
+
+    //    public abstract StorageMetaData getNewStorageMetaData();
+    public abstract StorageMetaData getNewStorageMetaData(long initTime, String sessionID, String workerID);
+    //                                                          Class<? extends StatsInitializationReport> initializationReportClass,
+    //                                                          Class<? extends StatsReport> statsReportClass);
+    //new SbeStorageMetaData(initTime, getSessionID(model), TYPE_ID, workerID, SbeStatsInitializationReport.class, SbeStatsReport.class);
+
+
+    public StatsInitializationConfiguration getInitConfig() {
+        return initConfig;
+    }
+
+    public StatsUpdateConfiguration getUpdateConfig() {
+        return updateConfig;
+    }
+
+    public void setUpdateConfig(StatsUpdateConfiguration newConfig) {
+        this.updateConfig = newConfig;
+    }
+
+    @Override
+    public void setStorageRouter(StatsStorageRouter router) {
+        this.router = router;
+    }
+
+    @Override
+    public StatsStorageRouter getStorageRouter() {
+        return router;
+    }
+
+    @Override
+    public void setWorkerID(String workerID) {
+        this.workerID = workerID;
+    }
+
+    @Override
+    public String getWorkerID() {
+        return workerID;
+    }
+
+    @Override
+    public void setSessionID(String sessionID) {
+        this.sessionID = sessionID;
+    }
+
+    @Override
+    public String getSessionID() {
+        return sessionID;
+    }
+
+    private String getSessionID(Model model) {
+        if (model instanceof MultiLayerNetwork || model instanceof ComputationGraph)
+            return sessionID;
+        if (model instanceof Layer) {
+            //Keep in mind MultiLayerNetwork implements Layer also...
+            Layer l = (Layer) model;
+            int layerIdx = l.getIndex();
+            return sessionID + "_layer" + layerIdx;
+        }
+        return sessionID; //Should never happen
+    }
+
+    @Override
+    public void onEpochStart(Model model) {
+
+    }
+
+    @Override
+    public void onEpochEnd(Model model) {
+
+    }
+
+    @Override
+    public void onForwardPass(Model model, List<INDArray> activations) {
+        int iterCount = getModelInfo(model).iterCount;
+        if (calcFromActivations() && (iterCount == 0 || iterCount % updateConfig.reportingFrequency() == 0)) {
+            //Assumption: we have input, layer 0, layer 1, ...
+            Map<String, INDArray> activationsMap = new HashMap<>();
+            int count = 0;
+            for (INDArray arr : activations) {
+                String layerName = (count == 0 ? "input" : String.valueOf(count - 1));
+                activationsMap.put(layerName, arr);
+                count++;
+            }
+            onForwardPass(model, activationsMap);
+        }
+    }
+
+    @Override
+    public void onForwardPass(Model model, Map<String, INDArray> activations) {
+        int iterCount = getModelInfo(model).iterCount;
+        if (calcFromActivations() && updateConfig.reportingFrequency() > 0
+                && (iterCount == 0 || iterCount % updateConfig.reportingFrequency() == 0)) {
+            if (updateConfig.collectHistograms(StatsType.Activations)) {
+                activationHistograms = getHistograms(activations, updateConfig.numHistogramBins(StatsType.Activations));
+            }
+            if (updateConfig.collectMean(StatsType.Activations)) {
+                meanActivations = calculateSummaryStats(activations, StatType.Mean);
+            }
+            if (updateConfig.collectStdev(StatsType.Activations)) {
+                stdevActivations = calculateSummaryStats(activations, StatType.Stdev);
+            }
+            if (updateConfig.collectMeanMagnitudes(StatsType.Activations)) {
+                meanMagActivations = calculateSummaryStats(activations, StatType.MeanMagnitude);
+            }
+        }
+    }
+
+    @Override
+    public void onGradientCalculation(Model model) {
+        int iterCount = getModelInfo(model).iterCount;
+        if (calcFromGradients() && updateConfig.reportingFrequency() > 0
+                && (iterCount == 0 || iterCount % updateConfig.reportingFrequency() == 0)) {
+            Gradient g = model.gradient();
+            if (updateConfig.collectHistograms(StatsType.Gradients)) {
+                gradientHistograms = getHistograms(g.gradientForVariable(), updateConfig.numHistogramBins(StatsType.Gradients));
+            }
+
+            if (updateConfig.collectMean(StatsType.Gradients)) {
+                meanGradients = calculateSummaryStats(g.gradientForVariable(), StatType.Mean);
+            }
+            if (updateConfig.collectStdev(StatsType.Gradients)) {
+                stdevGradient = calculateSummaryStats(g.gradientForVariable(), StatType.Stdev);
+            }
+            if (updateConfig.collectMeanMagnitudes(StatsType.Gradients)) {
+                meanMagGradients = calculateSummaryStats(g.gradientForVariable(), StatType.MeanMagnitude);
+            }
+        }
+    }
+
+    private boolean calcFromActivations() {
+        return updateConfig.collectMean(StatsType.Activations) || updateConfig.collectStdev(StatsType.Activations)
+                || updateConfig.collectMeanMagnitudes(StatsType.Activations)
+                || updateConfig.collectHistograms(StatsType.Activations);
+    }
+
+    private boolean calcFromGradients() {
+        return updateConfig.collectMean(StatsType.Gradients) || updateConfig.collectStdev(StatsType.Gradients)
+                || updateConfig.collectMeanMagnitudes(StatsType.Gradients)
+                || updateConfig.collectHistograms(StatsType.Gradients);
+    }
+
+    @Override
+    public void onBackwardPass(Model model) {
+        //No op
+    }
+
+    @Override
+    public void iterationDone(Model model, int iteration, int epoch) {
+
+        ModelInfo modelInfo = getModelInfo(model);
+        boolean backpropParamsOnly = backpropParamsOnly(model);
+
+        long currentTime = getTime();
+        if (modelInfo.iterCount == 0) {
+            modelInfo.initTime = currentTime;
+            doInit(model);
+        }
+
+        if (updateConfig.collectPerformanceStats()) {
+            updateExamplesMinibatchesCounts(model);
+        }
+
+        if (updateConfig.reportingFrequency() > 1 && (iteration == 0 || iteration % updateConfig.reportingFrequency() != 0)) {
+            modelInfo.iterCount = iteration;
+            return;
+        }
+
+        StatsReport report = getNewStatsReport();
+        report.reportIDs(getSessionID(model), TYPE_ID, workerID, System.currentTimeMillis()); //TODO support NTP time
+
+        //--- Performance and System Stats ---
+        if (updateConfig.collectPerformanceStats()) {
+            //Stats to collect: total runtime, total examples, total minibatches, iterations/second, examples/second
+            double examplesPerSecond;
+            double minibatchesPerSecond;
+            if (modelInfo.iterCount == 0) {
+                //Not possible to work out perf/second: first iteration...
+                examplesPerSecond = 0.0;
+                minibatchesPerSecond = 0.0;
+            } else {
+                long deltaTimeMS = currentTime - modelInfo.lastReportTime;
+                examplesPerSecond = 1000.0 * modelInfo.examplesSinceLastReport / deltaTimeMS;
+                minibatchesPerSecond = 1000.0 * modelInfo.minibatchesSinceLastReport / deltaTimeMS;
+            }
+            long totalRuntimeMS = currentTime - modelInfo.initTime;
+            report.reportPerformance(totalRuntimeMS, modelInfo.totalExamples, modelInfo.totalMinibatches,
+                    examplesPerSecond, minibatchesPerSecond);
+
+            modelInfo.examplesSinceLastReport = 0;
+            modelInfo.minibatchesSinceLastReport = 0;
+        }
+
+        if (updateConfig.collectMemoryStats()) {
+
+            Runtime runtime = Runtime.getRuntime();
+            long jvmTotal = runtime.totalMemory();
+            long jvmMax = runtime.maxMemory();
+
+            //Off-heap memory
+            long offheapTotal = Pointer.totalBytes();
+            long offheapMax = Pointer.maxBytes();
+
+            //GPU
+            long[] gpuCurrentBytes = null;
+            long[] gpuMaxBytes = null;
+            NativeOps nativeOps =Nd4j.getNativeOps();
+            int nDevices = nativeOps.getAvailableDevices();
+            if (nDevices > 0) {
+                gpuCurrentBytes = new long[nDevices];
+                gpuMaxBytes = new long[nDevices];
+                for (int i = 0; i < nDevices; i++) {
+                    try {
+                        gpuMaxBytes[i] = nativeOps.getDeviceTotalMemory(0);
+                        gpuCurrentBytes[i] = gpuMaxBytes[i] - nativeOps.getDeviceFreeMemory(0);
+                    } catch (Exception e) {
+                        log.error("",e);
+                    }
+                }
+            }
+
+            report.reportMemoryUse(jvmTotal, jvmMax, offheapTotal, offheapMax, gpuCurrentBytes, gpuMaxBytes);
+        }
+
+        if (updateConfig.collectGarbageCollectionStats()) {
+            if (modelInfo.lastReportIteration == -1 || gcBeans == null) {
+                //Haven't reported GC stats before...
+                gcBeans = ManagementFactory.getGarbageCollectorMXBeans();
+                gcStatsAtLastReport = new HashMap<>();
+                for (GarbageCollectorMXBean bean : gcBeans) {
+                    long count = bean.getCollectionCount();
+                    long timeMs = bean.getCollectionTime();
+                    gcStatsAtLastReport.put(bean.getName(), new Pair<>(count, timeMs));
+                }
+            } else {
+                for (GarbageCollectorMXBean bean : gcBeans) {
+                    long count = bean.getCollectionCount();
+                    long timeMs = bean.getCollectionTime();
+                    Pair<Long, Long> lastStats = gcStatsAtLastReport.get(bean.getName());
+                    long deltaGCCount = count - lastStats.getFirst();
+                    long deltaGCTime = timeMs - lastStats.getSecond();
+
+                    lastStats.setFirst(count);
+                    lastStats.setSecond(timeMs);
+                    report.reportGarbageCollection(bean.getName(), (int) deltaGCCount, (int) deltaGCTime);
+                }
+            }
+        }
+
+        //--- General ---
+        report.reportScore(model.score()); //Always report score
+
+        if (updateConfig.collectLearningRates()) {
+            Map<String, Double> lrs = new HashMap<>();
+            if (model instanceof MultiLayerNetwork) {
+                //Need to append "0_", "1_" etc to param names from layers...
+                int layerIdx = 0;
+                for (Layer l : ((MultiLayerNetwork) model).getLayers()) {
+                    NeuralNetConfiguration conf = l.conf();
+                    List<String> paramkeys = l.conf().getLayer().initializer().paramKeys(l.conf().getLayer());
+                    for (String s : paramkeys) {
+                        double lr = conf.getLayer().getUpdaterByParam(s).getLearningRate(l.getIterationCount(), l.getEpochCount());
+                        if (Double.isNaN(lr)) {
+                            //Edge case: No-Op updater, AdaDelta etc - don't have a LR hence return NaN for IUpdater.getLearningRate
+                            lr = 0.0;
+                        }
+                        lrs.put(layerIdx + "_" + s, lr);
+                    }
+                    layerIdx++;
+                }
+            } else if (model instanceof ComputationGraph) {
+                for (Layer l : ((ComputationGraph) model).getLayers()) {
+                    NeuralNetConfiguration conf = l.conf();
+                    String layerName = conf.getLayer().getLayerName();
+                    List<String> paramkeys = l.conf().getLayer().initializer().paramKeys(l.conf().getLayer());
+                    for (String s : paramkeys) {
+                        double lr = conf.getLayer().getUpdaterByParam(s).getLearningRate(l.getIterationCount(), l.getEpochCount());
+                        if (Double.isNaN(lr)) {
+                            //Edge case: No-Op updater, AdaDelta etc - don't have a LR hence return NaN for IUpdater.getLearningRate
+                            lr = 0.0;
+                        }
+                        lrs.put(layerName + "_" + s, lr);
+                    }
+                }
+            } else if (model instanceof Layer) {
+                Layer l = (Layer) model;
+                List<String> paramkeys = l.conf().getLayer().initializer().paramKeys(l.conf().getLayer());
+                for (String s : paramkeys) {
+                    double lr = l.conf().getLayer().getUpdaterByParam(s).getLearningRate(l.getIterationCount(), l.getEpochCount());
+                    lrs.put(s, lr);
+                }
+            }
+            report.reportLearningRates(lrs);
+        }
+
+
+        //--- Histograms ---
+
+        if (updateConfig.collectHistograms(StatsType.Parameters)) {
+            Map<String, Histogram> paramHistograms = getHistograms(model.paramTable(backpropParamsOnly),
+                    updateConfig.numHistogramBins(StatsType.Parameters));
+            report.reportHistograms(StatsType.Parameters, paramHistograms);
+        }
+
+        if (updateConfig.collectHistograms(StatsType.Gradients)) {
+            report.reportHistograms(StatsType.Gradients, gradientHistograms);
+        }
+
+        if (updateConfig.collectHistograms(StatsType.Updates)) {
+            Map<String, Histogram> updateHistograms = getHistograms(model.gradient().gradientForVariable(),
+                    updateConfig.numHistogramBins(StatsType.Updates));
+            report.reportHistograms(StatsType.Updates, updateHistograms);
+        }
+
+        if (updateConfig.collectHistograms(StatsType.Activations)) {
+            report.reportHistograms(StatsType.Activations, activationHistograms);
+        }
+
+
+        //--- Summary Stats: Mean, Variance, Mean Magnitudes ---
+
+        if (updateConfig.collectMean(StatsType.Parameters)) {
+            Map<String, Double> meanParams = calculateSummaryStats(model.paramTable(backpropParamsOnly), StatType.Mean);
+            report.reportMean(StatsType.Parameters, meanParams);
+        }
+
+        if (updateConfig.collectMean(StatsType.Gradients)) {
+            report.reportMean(StatsType.Gradients, meanGradients);
+        }
+
+        if (updateConfig.collectMean(StatsType.Updates)) {
+            Map<String, Double> meanUpdates =
+                    calculateSummaryStats(model.gradient().gradientForVariable(), StatType.Mean);
+            report.reportMean(StatsType.Updates, meanUpdates);
+        }
+
+        if (updateConfig.collectMean(StatsType.Activations)) {
+            report.reportMean(StatsType.Activations, meanActivations);
+        }
+
+
+        if (updateConfig.collectStdev(StatsType.Parameters)) {
+            Map<String, Double> stdevParams =
+                    calculateSummaryStats(model.paramTable(backpropParamsOnly), StatType.Stdev);
+            report.reportStdev(StatsType.Parameters, stdevParams);
+        }
+
+        if (updateConfig.collectStdev(StatsType.Gradients)) {
+            report.reportStdev(StatsType.Gradients, stdevGradient);
+        }
+
+        if (updateConfig.collectStdev(StatsType.Updates)) {
+            Map<String, Double> stdevUpdates =
+                    calculateSummaryStats(model.gradient().gradientForVariable(), StatType.Stdev);
+            report.reportStdev(StatsType.Updates, stdevUpdates);
+        }
+
+        if (updateConfig.collectStdev(StatsType.Activations)) {
+            report.reportStdev(StatsType.Activations, stdevActivations);
+        }
+
+
+        if (updateConfig.collectMeanMagnitudes(StatsType.Parameters)) {
+            Map<String, Double> meanMagParams =
+                    calculateSummaryStats(model.paramTable(backpropParamsOnly), StatType.MeanMagnitude);
+            report.reportMeanMagnitudes(StatsType.Parameters, meanMagParams);
+        }
+
+        if (updateConfig.collectMeanMagnitudes(StatsType.Gradients)) {
+            report.reportMeanMagnitudes(StatsType.Gradients, meanMagGradients);
+        }
+
+        if (updateConfig.collectMeanMagnitudes(StatsType.Updates)) {
+            Map<String, Double> meanMagUpdates =
+                    calculateSummaryStats(model.gradient().gradientForVariable(), StatType.MeanMagnitude);
+            report.reportMeanMagnitudes(StatsType.Updates, meanMagUpdates);
+        }
+
+        if (updateConfig.collectMeanMagnitudes(StatsType.Activations)) {
+            report.reportMeanMagnitudes(StatsType.Activations, meanMagActivations);
+        }
+
+
+        long endTime = getTime();
+        report.reportStatsCollectionDurationMS((int) (endTime - currentTime)); //Amount of time required to alculate all histograms, means etc.
+        modelInfo.lastReportTime = currentTime;
+        modelInfo.lastReportIteration = iteration;
+        report.reportIterationCount(iteration);
+
+        this.router.putUpdate(report);
+
+        modelInfo.iterCount = iteration;
+        activationHistograms = null;
+        meanActivations = null;
+        stdevActivations = null;
+        meanMagActivations = null;
+        gradientHistograms = null;
+        meanGradients = null;
+        stdevGradient = null;
+        meanMagGradients = null;
+    }
+
+    private long getTime() {
+        //Abstraction to allow NTP to be plugged in later...
+        return System.currentTimeMillis();
+    }
+
+    private void doInit(Model model) {
+        boolean backpropParamsOnly = backpropParamsOnly(model);
+        long initTime = System.currentTimeMillis(); //TODO support NTP
+        StatsInitializationReport initReport = getNewInitializationReport();
+        initReport.reportIDs(getSessionID(model), TYPE_ID, workerID, initTime);
+
+        if (initConfig.collectSoftwareInfo()) {
+            OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean();
+            RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
+
+            String arch = osBean.getArch();
+            String osName = osBean.getName();
+            String jvmName = runtime.getVmName();
+            String jvmVersion = System.getProperty("java.version");
+            String jvmSpecVersion = runtime.getSpecVersion();
+
+            String nd4jBackendClass = Nd4j.getNDArrayFactory().getClass().getName();
+            String nd4jDataTypeName = DataTypeUtil.getDtypeFromContext().name();
+
+            String hostname = System.getenv("COMPUTERNAME");
+            if (hostname == null || hostname.isEmpty()) {
+                try {
+                    Process proc = Runtime.getRuntime().exec("hostname");
+                    try (InputStream stream = proc.getInputStream()) {
+                        hostname = IOUtils.toString(stream);
+                    }
+                } catch (Exception e) {
+                }
+            }
+
+            Properties p = Nd4j.getExecutioner().getEnvironmentInformation();
+            Map<String, String> envInfo = new HashMap<>();
+            for (Map.Entry<Object, Object> e : p.entrySet()) {
+                Object v = e.getValue();
+                String value = (v == null ? "" : v.toString());
+                envInfo.put(e.getKey().toString(), value);
+            }
+
+            initReport.reportSoftwareInfo(arch, osName, jvmName, jvmVersion, jvmSpecVersion, nd4jBackendClass,
+                    nd4jDataTypeName, hostname, UIDProvider.getJVMUID(), envInfo);
+        }
+
+        if (initConfig.collectHardwareInfo()) {
+            int availableProcessors = Runtime.getRuntime().availableProcessors();
+            NativeOps nativeOps =Nd4j.getNativeOps();
+            int nDevices = nativeOps.getAvailableDevices();
+
+            long[] deviceTotalMem = null;
+            String[] deviceDescription = null; //TODO
+            if (nDevices > 0) {
+                deviceTotalMem = new long[nDevices];
+                deviceDescription = new String[nDevices];
+                for (int i = 0; i < nDevices; i++) {
+                    try {
+                        deviceTotalMem[i] = nativeOps.getDeviceTotalMemory(i);
+                        deviceDescription[i] = nativeOps.getDeviceName(i);
+                        if (nDevices > 1) {
+                            deviceDescription[i] = deviceDescription[i] + " (" + i + ")";
+                        }
+                    } catch (Exception e) {
+                        log.debug("Error getting device info", e);
+                    }
+                }
+            }
+            long jvmMaxMemory = Runtime.getRuntime().maxMemory();
+            long offheapMaxMemory = Pointer.maxBytes();
+
+            initReport.reportHardwareInfo(availableProcessors, nDevices, jvmMaxMemory, offheapMaxMemory, deviceTotalMem,
+                    deviceDescription, UIDProvider.getHardwareUID());
+        }
+
+        if (initConfig.collectModelInfo()) {
+            String jsonConf;
+            int numLayers;
+            long numParams;
+            if (model instanceof MultiLayerNetwork) {
+                MultiLayerNetwork net = ((MultiLayerNetwork) model);
+                jsonConf = net.getLayerWiseConfigurations().toJson();
+                numLayers = net.getnLayers();
+                numParams = net.numParams();
+            } else if (model instanceof ComputationGraph) {
+                ComputationGraph cg = ((ComputationGraph) model);
+                jsonConf = cg.getConfiguration().toJson();
+                numLayers = cg.getNumLayers();
+                numParams = cg.numParams();
+            } else if (model instanceof Layer) {
+                Layer l = (Layer) model;
+                jsonConf = l.conf().toJson();
+                numLayers = 1;
+                numParams = l.numParams();
+            } else {
+                throw new RuntimeException("Invalid model: Expected MultiLayerNetwork or ComputationGraph. Got: "
+                        + (model == null ? null : model.getClass()));
+            }
+
+            Map<String, INDArray> paramMap = model.paramTable(backpropParamsOnly);
+            String[] paramNames = new String[paramMap.size()];
+            int i = 0;
+            for (String s : paramMap.keySet()) { //Assuming sensible iteration order - LinkedHashMaps are used in MLN/CG for example
+                paramNames[i++] = s;
+            }
+
+            initReport.reportModelInfo(model.getClass().getName(), jsonConf, paramNames, numLayers, numParams);
+        }
+
+        StorageMetaData meta = getNewStorageMetaData(initTime, getSessionID(model), workerID);
+
+        router.putStorageMetaData(meta);
+        router.putStaticInfo(initReport); //TODO error handling
+    }
+
+    private Map<Integer, Pointer> devPointers = new HashMap<>();
+
+    private synchronized Pointer getDevicePointer(int device) {
+        if (devPointers.containsKey(device)) {
+            return devPointers.get(device);
+        }
+        try {
+            Pointer pointer = DL4JClassLoading.createNewInstance(
+                    "org.nd4j.jita.allocator.pointers.CudaPointer",
+                    Pointer.class,
+                    new Class[] { long.class },
+                    new Object[]{(long) device});
+
+            devPointers.put(device, pointer);
+            return pointer;
+        } catch (Throwable t) {
+            devPointers.put(device, null); //Stops attempting the failure again later...
+            return null;
+        }
+    }
+
+    private void updateExamplesMinibatchesCounts(Model model) {
+        ModelInfo modelInfo = getModelInfo(model);
+        int examplesThisMinibatch = 0;
+        if (model instanceof MultiLayerNetwork) {
+            examplesThisMinibatch = model.batchSize();
+        } else if (model instanceof ComputationGraph) {
+            examplesThisMinibatch = model.batchSize();
+        } else if (model instanceof Layer) {
+            examplesThisMinibatch = ((Layer) model).getInputMiniBatchSize();
+        }
+        modelInfo.examplesSinceLastReport += examplesThisMinibatch;
+        modelInfo.totalExamples += examplesThisMinibatch;
+        modelInfo.minibatchesSinceLastReport++;
+        modelInfo.totalMinibatches++;
+    }
+
+    private boolean backpropParamsOnly(Model model) {
+        //For pretrain layers (VAE, AE) we *do* want pretrain params also; for MLN and CG we only want backprop params
+        // as we only have backprop gradients
+        return model instanceof MultiLayerNetwork || model instanceof ComputationGraph;
+    }
+
+    private static Map<String, Double> calculateSummaryStats(Map<String, INDArray> source, StatType statType) {
+        Map<String, Double> out = new LinkedHashMap<>();
+
+        if (source == null)
+            return out;
+
+        for (Map.Entry<String, INDArray> entry : source.entrySet()) {
+            String name = entry.getKey();
+            double value;
+            switch (statType) {
+                case Mean:
+                    value = entry.getValue().meanNumber().doubleValue();
+                    break;
+                case Stdev:
+                    value = entry.getValue().stdNumber().doubleValue();
+                    break;
+                case MeanMagnitude:
+                    value = entry.getValue().norm1Number().doubleValue() / entry.getValue().length();
+                    break;
+                default:
+                    throw new RuntimeException(); //Should never happen
+            }
+            out.put(name, value);
+        }
+        return out;
+    }
+
+    private static Map<String, Histogram> getHistograms(Map<String, INDArray> map, int nBins) {
+        Map<String, Histogram> out = new LinkedHashMap<>();
+
+        if (map == null)
+            return out;
+
+        for (Map.Entry<String, INDArray> entry : map.entrySet()) {
+
+            org.nd4j.linalg.api.ops.impl.transforms.Histogram hOp =
+                    new org.nd4j.linalg.api.ops.impl.transforms.Histogram(entry.getValue(), nBins);
+            Nd4j.exec(hOp);
+
+            INDArray bins = hOp.getOutputArgument(0);
+            int[] count = new int[nBins];
+            for (int i = 0; i < bins.length(); i++) {
+                count[i] = (int) bins.getDouble(i);
+            }
+
+            double min = entry.getValue().minNumber().doubleValue();
+            double max = entry.getValue().maxNumber().doubleValue();
+
+            Histogram h = new Histogram(min, max, nBins, count);
+
+            out.put(entry.getKey(), h);
+        }
+        return out;
+    }
+
+    @Override
+    public abstract BaseStatsListener clone();
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/J7StatsListener.java

@@ -0,0 +1,88 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats;
+
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.core.storage.StatsStorageRouter;
+import org.deeplearning4j.core.storage.StorageMetaData;
+import org.deeplearning4j.ui.model.stats.api.StatsInitializationConfiguration;
+import org.deeplearning4j.ui.model.stats.api.StatsInitializationReport;
+import org.deeplearning4j.ui.model.stats.api.StatsReport;
+import org.deeplearning4j.ui.model.stats.api.StatsUpdateConfiguration;
+import org.deeplearning4j.ui.model.storage.FileStatsStorage;
+import org.deeplearning4j.ui.model.storage.InMemoryStatsStorage;
+import org.deeplearning4j.ui.model.storage.impl.JavaStorageMetaData;
+import org.deeplearning4j.ui.model.stats.impl.DefaultStatsUpdateConfiguration;
+import org.deeplearning4j.ui.model.stats.impl.java.JavaStatsInitializationReport;
+import org.deeplearning4j.ui.model.stats.impl.java.JavaStatsReport;
+
+@Slf4j
+public class J7StatsListener extends BaseStatsListener {
+
+    /**
+     * Create a StatsListener with network information collected at every iteration. Equivalent to {@link #J7StatsListener(StatsStorageRouter, int)}
+     * with {@code listenerFrequency == 1}
+     *
+     * @param router Where/how to store the calculated stats. For example, {@link InMemoryStatsStorage} or
+     *               {@link FileStatsStorage}
+     */
+    public J7StatsListener(StatsStorageRouter router) {
+        this(router, null, null, null, null);
+    }
+
+    /**
+     * Create a StatsListener with network information collected every n >= 1 time steps
+     *
+     * @param router            Where/how to store the calculated stats. For example, {@link InMemoryStatsStorage} or
+     *                          {@link FileStatsStorage}
+     * @param listenerFrequency Frequency with which to collect stats information
+     */
+    public J7StatsListener(StatsStorageRouter router, int listenerFrequency) {
+        this(router, null, new DefaultStatsUpdateConfiguration.Builder().reportingFrequency(listenerFrequency).build(),
+                        null, null);
+    }
+
+    public J7StatsListener(StatsStorageRouter router, StatsInitializationConfiguration initConfig,
+                           StatsUpdateConfiguration updateConfig, String sessionID, String workerID) {
+        super(router, initConfig, updateConfig, sessionID, workerID);
+    }
+
+    @Override
+    public StatsInitializationReport getNewInitializationReport() {
+        return new JavaStatsInitializationReport();
+    }
+
+    @Override
+    public StatsReport getNewStatsReport() {
+        return new JavaStatsReport();
+    }
+
+    @Override
+    public StorageMetaData getNewStorageMetaData(long initTime, String sessionID, String workerID) {
+        return new JavaStorageMetaData(initTime, sessionID, TYPE_ID, workerID,
+                        JavaStatsInitializationReport.class, JavaStatsReport.class);
+    }
+
+    @Override
+    public J7StatsListener clone() {
+        return new J7StatsListener(this.getStorageRouter(), this.getInitConfig(), this.getUpdateConfig(), null, null);
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/StatsListener.java

@@ -0,0 +1,99 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats;
+
+import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.core.storage.StatsStorageRouter;
+import org.deeplearning4j.core.storage.StorageMetaData;
+import org.deeplearning4j.ui.model.stats.api.StatsInitializationConfiguration;
+import org.deeplearning4j.ui.model.stats.api.StatsInitializationReport;
+import org.deeplearning4j.ui.model.stats.api.StatsReport;
+import org.deeplearning4j.ui.model.stats.api.StatsUpdateConfiguration;
+import org.deeplearning4j.ui.model.storage.FileStatsStorage;
+import org.deeplearning4j.ui.model.storage.InMemoryStatsStorage;
+import org.deeplearning4j.ui.model.storage.impl.SbeStorageMetaData;
+import org.deeplearning4j.ui.model.stats.impl.DefaultStatsUpdateConfiguration;
+import org.deeplearning4j.ui.model.stats.impl.SbeStatsInitializationReport;
+import org.deeplearning4j.ui.model.stats.impl.SbeStatsReport;
+
+@Slf4j
+public class StatsListener extends BaseStatsListener {
+
+    /**
+     * Create a StatsListener with network information collected at every iteration. Equivalent to {@link #StatsListener(StatsStorageRouter, int)}
+     * with {@code listenerFrequency == 1}
+     *
+     * @param router Where/how to store the calculated stats. For example, {@link InMemoryStatsStorage} or
+     *               {@link FileStatsStorage}
+     */
+    public StatsListener(StatsStorageRouter router) {
+        this(router, null, null, null, null);
+    }
+
+    /**
+     * Create a StatsListener with network information collected every n >= 1 time steps
+     *
+     * @param router            Where/how to store the calculated stats. For example, {@link InMemoryStatsStorage} or
+     *                          {@link FileStatsStorage}
+     * @param listenerFrequency Frequency with which to collect stats information
+     */
+    public StatsListener(StatsStorageRouter router, int listenerFrequency) {
+        this(router, listenerFrequency, null);
+    }
+
+    /**
+     * Create a StatsListener with network information collected every n >= 1 time steps
+     *
+     * @param router            Where/how to store the calculated stats. For example, {@link InMemoryStatsStorage} or
+     *                          {@link FileStatsStorage}
+     * @param listenerFrequency Frequency with which to collect stats information
+     * @param sessionId         The Session ID for storing the stats, optional (may be null)
+     */
+    public StatsListener(StatsStorageRouter router, int listenerFrequency, String sessionId) {
+        this(router, null, new DefaultStatsUpdateConfiguration.Builder().reportingFrequency(listenerFrequency).build(),
+                sessionId, null);
+    }
+
+    public StatsListener(StatsStorageRouter router, StatsInitializationConfiguration initConfig,
+                         StatsUpdateConfiguration updateConfig, String sessionID, String workerID) {
+        super(router, initConfig, updateConfig, sessionID, workerID);
+    }
+
+    public StatsListener clone() {
+        return new StatsListener(this.getStorageRouter(), this.getInitConfig(), this.getUpdateConfig(), null, null);
+    }
+
+    @Override
+    public StatsInitializationReport getNewInitializationReport() {
+        return new SbeStatsInitializationReport();
+    }
+
+    @Override
+    public StatsReport getNewStatsReport() {
+        return new SbeStatsReport();
+    }
+
+    @Override
+    public StorageMetaData getNewStorageMetaData(long initTime, String sessionID, String workerID) {
+        return new SbeStorageMetaData(initTime, sessionID, BaseStatsListener.TYPE_ID, workerID,
+                        SbeStatsInitializationReport.class, SbeStatsReport.class);
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/Histogram.java

@@ -0,0 +1,37 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.api;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.io.Serializable;
+
+@AllArgsConstructor
+@NoArgsConstructor
+@Data
+public class Histogram implements Serializable {
+    private double min;
+    private double max;
+    private int nBins;
+    private int[] binCounts;
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsInitializationConfiguration.java

@@ -0,0 +1,50 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.api;
+
+import org.deeplearning4j.ui.model.stats.StatsListener;
+
+import java.io.Serializable;
+
+public interface StatsInitializationConfiguration extends Serializable {
+
+    /**
+     * Should software configuration information be collected? For example, OS, JVM, and ND4J backend details
+     *
+     * @return true if software information should be collected; false if not
+     */
+    boolean collectSoftwareInfo();
+
+    /**
+     * Should hardware configuration information be collected? JVM available processors, number of devices, total memory for each device
+     *
+     * @return true if hardware information should be collected
+     */
+    boolean collectHardwareInfo();
+
+    /**
+     * Should model information be collected? Model class, configuration (JSON), number of layers, number of parameters, etc.
+     *
+     * @return true if model information should be collected
+     */
+    boolean collectModelInfo();
+
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsInitializationReport.java

@@ -0,0 +1,126 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.api;
+
+import org.deeplearning4j.core.storage.Persistable;
+import org.deeplearning4j.ui.model.stats.StatsListener;
+
+import java.util.Map;
+
+public interface StatsInitializationReport extends Persistable {
+
+    void reportIDs(String sessionID, String typeID, String workerID, long timestamp);
+
+    /**
+     * @param arch             Operating system architecture, as reported by JVM
+     * @param osName           Operating system name
+     * @param jvmName          JVM name
+     * @param jvmVersion       JVM version
+     * @param jvmSpecVersion   JVM Specification version (for example, 1.8)
+     * @param nd4jBackendClass ND4J backend Factory class
+     * @param nd4jDataTypeName ND4J datatype name
+     * @param hostname         Hostname for the machine, if available
+     * @param jvmUID           A unique identified for the current JVM. Should be shared by all instances in the same JVM.
+     *                         Should vary for different JVMs on the same machine.
+     * @param swEnvironmentInfo Environment information: Usually from Nd4j.getExecutioner().getEnvironmentInformation()
+     */
+    void reportSoftwareInfo(String arch, String osName, String jvmName, String jvmVersion, String jvmSpecVersion,
+                    String nd4jBackendClass, String nd4jDataTypeName, String hostname, String jvmUID,
+                    Map<String, String> swEnvironmentInfo);
+
+    /**
+     * @param jvmAvailableProcessors Number of available processor cores according to the JVM
+     * @param numDevices             Number of compute devices (GPUs)
+     * @param jvmMaxMemory           Maximum memory for the JVM
+     * @param offHeapMaxMemory       Maximum off-heap memory
+     * @param deviceTotalMemory      GPU memory by device: same length as numDevices. May be null, if numDevices is 0
+     * @param deviceDescription      Description of each device. May be null, if numDevices is 0
+     * @param hardwareUID            A unique identifier for the machine. Should be shared by all instances running on
+     *                               the same machine, including in different JVMs
+     *
+     */
+    void reportHardwareInfo(int jvmAvailableProcessors, int numDevices, long jvmMaxMemory, long offHeapMaxMemory,
+                    long[] deviceTotalMemory, String[] deviceDescription, String hardwareUID);
+
+
+    /**
+     * Report the model information
+     *
+     * @param modelClassName  Model class name: i.e., type of model
+     * @param modelConfigJson Model configuration, as JSON string
+     * @param numLayers       Number of layers in the model
+     * @param numParams       Number of parameters in the model
+     */
+    void reportModelInfo(String modelClassName, String modelConfigJson, String[] paramNames, int numLayers,
+                    long numParams);
+
+
+    boolean hasSoftwareInfo();
+
+    boolean hasHardwareInfo();
+
+    boolean hasModelInfo();
+
+    String getSwArch();
+
+    String getSwOsName();
+
+    String getSwJvmName();
+
+    String getSwJvmVersion();
+
+    String getSwJvmSpecVersion();
+
+    String getSwNd4jBackendClass();
+
+    String getSwNd4jDataTypeName();
+
+    String getSwHostName();
+
+    String getSwJvmUID();
+
+    Map<String, String> getSwEnvironmentInfo();
+
+    int getHwJvmAvailableProcessors();
+
+    int getHwNumDevices();
+
+    long getHwJvmMaxMemory();
+
+    long getHwOffHeapMaxMemory();
+
+    long[] getHwDeviceTotalMemory();
+
+    String[] getHwDeviceDescription();
+
+    String getHwHardwareUID();
+
+    String getModelClassName();
+
+    String getModelConfigJson();
+
+    String[] getModelParamNames();
+
+    int getModelNumLayers();
+
+    long getModelNumParams();
+
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsReport.java

@@ -0,0 +1,322 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.api;
+
+import org.deeplearning4j.core.storage.Persistable;
+import org.deeplearning4j.ui.model.stats.StatsListener;
+import org.nd4j.common.primitives.Pair;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+public interface StatsReport extends Persistable {
+
+    void reportIDs(String sessionID, String typeID, String workerID, long timestamp);
+
+    /**
+     * Report the current iteration number
+     */
+    void reportIterationCount(int iterationCount);
+
+    /**
+     * Get the current iteration number
+     */
+    int getIterationCount();
+
+    /**
+     * Report the number of milliseconds required to calculate all of the stats. This is effectively the
+     * amount of listener overhead
+     */
+    void reportStatsCollectionDurationMS(int statsCollectionDurationMS);
+
+    /**
+     * Get the number of millisecons required to calculate al of the stats. This is effectively the amount of
+     * listener overhead.
+     */
+    int getStatsCollectionDurationMs();
+
+    /**
+     * Report model score at the current iteration
+     */
+    void reportScore(double currentScore);
+
+    /**
+     * Get the score at the current iteration
+     */
+    double getScore();
+
+    /**
+     * Report the learning rates by parameter
+     */
+    void reportLearningRates(Map<String, Double> learningRatesByParam);
+
+    /**
+     * Get the learning rates by parameter
+     */
+    Map<String, Double> getLearningRates();
+
+
+    //--- Performance and System Stats ---
+
+    /**
+     * Report the memory stats at this iteration
+     *
+     * @param jvmCurrentBytes     Current bytes used by the JVM
+     * @param jvmMaxBytes         Max bytes usable by the JVM (heap)
+     * @param offHeapCurrentBytes Current off-heap bytes used
+     * @param offHeapMaxBytes     Maximum off-heap bytes
+     * @param deviceCurrentBytes  Current bytes used by each device (GPU, etc). May be null if no devices are present
+     * @param deviceMaxBytes      Maximum bytes for each device (GPU, etc). May be null if no devices are present
+     */
+    void reportMemoryUse(long jvmCurrentBytes, long jvmMaxBytes, long offHeapCurrentBytes, long offHeapMaxBytes,
+                    long[] deviceCurrentBytes, long[] deviceMaxBytes);
+
+    /**
+     * Get JVM memory - current bytes used
+     */
+    long getJvmCurrentBytes();
+
+    /**
+     * Get JVM memory - max available bytes
+     */
+    long getJvmMaxBytes();
+
+    /**
+     * Get off-heap memory - current bytes used
+     */
+    long getOffHeapCurrentBytes();
+
+    /**
+     * Get off-heap memory - max available bytes
+     */
+    long getOffHeapMaxBytes();
+
+    /**
+     * Get device (GPU, etc) current bytes - may be null if no compute devices are present in the system
+     */
+    long[] getDeviceCurrentBytes();
+
+    /**
+     * Get device (GPU, etc) maximum bytes - may be null if no compute devices are present in the system
+     */
+    long[] getDeviceMaxBytes();
+
+    /**
+     * Report the performance stats (since the last report)
+     *
+     * @param totalRuntimeMs       Overall runtime since initialization
+     * @param totalExamples        Total examples processed since initialization
+     * @param totalMinibatches     Total number of minibatches (iterations) since initialization
+     * @param examplesPerSecond    Examples per second since last report
+     * @param minibatchesPerSecond Minibatches per second since last report
+     */
+    void reportPerformance(long totalRuntimeMs, long totalExamples, long totalMinibatches, double examplesPerSecond,
+                    double minibatchesPerSecond);
+
+    /**
+     * Get the total runtime since listener/model initialization
+     */
+    long getTotalRuntimeMs();
+
+    /**
+     * Get total number of examples that have been processed since initialization
+     */
+    long getTotalExamples();
+
+    /**
+     * Get the total number of minibatches that have been processed since initialization
+     */
+    long getTotalMinibatches();
+
+    /**
+     * Get examples per second since the last report
+     */
+    double getExamplesPerSecond();
+
+    /**
+     * Get the number of minibatches per second, since the last report
+     */
+    double getMinibatchesPerSecond();
+
+    /**
+     * Report Garbage collection stats
+     *
+     * @param gcName       Garbage collector name
+     * @param deltaGCCount Change in the total number of garbage collections, since last report
+     * @param deltaGCTime  Change in the amount of time (milliseconds) for garbage collection, since last report
+     */
+    void reportGarbageCollection(String gcName, int deltaGCCount, int deltaGCTime);
+
+    /**
+     * Get the garbage collection stats: Pair contains GC name and the delta count/time values
+     */
+    List<Pair<String, int[]>> getGarbageCollectionStats();
+
+    //--- Histograms ---
+
+    /**
+     * Report histograms for all parameters, for a given {@link StatsType}
+     *
+     * @param statsType StatsType: Parameters, Updates, Activations
+     * @param histogram Histogram values for all parameters
+     */
+    void reportHistograms(StatsType statsType, Map<String, Histogram> histogram);
+
+    /**
+     * Get the histograms for all parameters, for a given StatsType (Parameters/Updates/Activations)
+     *
+     * @param statsType Stats type (Params/updatse/activations) to get histograms for
+     * @return Histograms by parameter name, or null if not available
+     */
+    Map<String, Histogram> getHistograms(StatsType statsType);
+
+    //--- Summary Stats: Mean, Variance, Mean Magnitudes ---
+
+    /**
+     * Report the mean values for each parameter, the given StatsType (Parameters/Updates/Activations)
+     *
+     * @param statsType Stats type to report
+     * @param mean      Map of mean values, by parameter
+     */
+    void reportMean(StatsType statsType, Map<String, Double> mean);
+
+    /**
+     * Get the mean values for each parameter for the given StatsType (Parameters/Updates/Activations)
+     *
+     * @param statsType Stats type to get mean values for
+     * @return Map of mean values by parameter
+     */
+    Map<String, Double> getMean(StatsType statsType);
+
+    /**
+     * Report the standard deviation values for each parameter for the given StatsType (Parameters/Updates/Activations)
+     *
+     * @param statsType Stats type to report std. dev values for
+     * @param stdev     Map of std dev values by parameter
+     */
+    void reportStdev(StatsType statsType, Map<String, Double> stdev);
+
+    /**
+     * Get the standard deviation values for each parameter for the given StatsType (Parameters/Updates/Activations)
+     *
+     * @param statsType Stats type to get std dev values for
+     * @return Map of stdev values by parameter
+     */
+    Map<String, Double> getStdev(StatsType statsType);
+
+    /**
+     * Report the mean magnitude values for each parameter for the given StatsType (Parameters/Updates/Activations)
+     *
+     * @param statsType      Stats type to report mean magnitude values for
+     * @param meanMagnitudes Map of mean magnitude values by parameter
+     */
+    void reportMeanMagnitudes(StatsType statsType, Map<String, Double> meanMagnitudes);
+
+    /**
+     * Report any metadata for the DataSet
+     *
+     * @param dataSetMetaData MetaData for the DataSet
+     * @param metaDataClass   Class of the metadata. Can be later retieved using {@link #getDataSetMetaDataClassName()}
+     */
+    void reportDataSetMetaData(List<Serializable> dataSetMetaData, Class<?> metaDataClass);
+
+    /**
+     * Report any metadata for the DataSet
+     *
+     * @param dataSetMetaData MetaData for the DataSet
+     * @param metaDataClass   Class of the metadata. Can be later retieved using {@link #getDataSetMetaDataClassName()}
+     */
+    void reportDataSetMetaData(List<Serializable> dataSetMetaData, String metaDataClass);
+
+    /**
+     * Get the mean magnitude values for each parameter for the given StatsType (Parameters/Updates/Activations)
+     *
+     * @param statsType Stats type to get mean magnitude values for
+     * @return Map of mean magnitude values by parameter
+     */
+    Map<String, Double> getMeanMagnitudes(StatsType statsType);
+
+    /**
+     * Get the DataSet metadata, if any (null otherwise).
+     * Note: due to serialization issues, this may in principle throw an unchecked exception related
+     * to class availability, serialization etc.
+     *
+     * @return List of DataSet metadata, if any.
+     */
+    List<Serializable> getDataSetMetaData();
+
+    /**
+     * Get the class
+     *
+     * @return
+     */
+    String getDataSetMetaDataClassName();
+
+    /**
+     * Return whether the score is present (has been reported)
+     */
+    boolean hasScore();
+
+    /**
+     * Return whether the learning rates are present (have been reported)
+     */
+    boolean hasLearningRates();
+
+    /**
+     * Return whether memory use has been reported
+     */
+    boolean hasMemoryUse();
+
+    /**
+     * Return whether performance stats (total time, total examples etc) have been reported
+     */
+    boolean hasPerformance();
+
+    /**
+     * Return whether garbage collection information has been reported
+     */
+    boolean hasGarbageCollection();
+
+    /**
+     * Return whether histograms have been reported, for the given stats type (Parameters, Updates, Activations)
+     *
+     * @param statsType Stats type
+     */
+    boolean hasHistograms(StatsType statsType);
+
+    /**
+     * Return whether the summary stats (mean, standard deviation, mean magnitudes) have been reported for the
+     * given stats type (Parameters, Updates, Activations)
+     *
+     * @param statsType   stats type (Parameters, Updates, Activations)
+     * @param summaryType Summary statistic type (mean, stdev, mean magnitude)
+     */
+    boolean hasSummaryStats(StatsType statsType, SummaryType summaryType);
+
+
+    /**
+     * Return whether any DataSet metadata is present or not
+     *
+     * @return True if DataSet metadata is present
+     */
+    boolean hasDataSetMetaData();
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsType.java

@@ -0,0 +1,29 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.api;
+
+import org.deeplearning4j.ui.model.stats.StatsListener;
+
+public enum StatsType {
+
+    Parameters, Gradients, Updates, Activations
+
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsUpdateConfiguration.java

@@ -0,0 +1,102 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.api;
+
+import java.io.Serializable;
+
+public interface StatsUpdateConfiguration extends Serializable {
+
+    /**
+     * Get the reporting frequency, in terms of listener calls
+     */
+    int reportingFrequency();
+
+    //TODO
+    //boolean useNTPTimeSource();
+
+    //--- Performance and System Stats ---
+
+    /**
+     * Should performance stats be collected/reported?
+     * Total time, total examples, total batches, Minibatches/second, examples/second
+     */
+    boolean collectPerformanceStats();
+
+    /**
+     * Should JVM, off-heap and memory stats be collected/reported?
+     */
+    boolean collectMemoryStats();
+
+    /**
+     * Should garbage collection stats be collected and reported?
+     */
+    boolean collectGarbageCollectionStats();
+
+    //TODO
+    //    boolean collectDataSetMetaData();
+
+    //--- General ---
+
+    /**
+     * Should per-parameter type learning rates be collected and reported?
+     */
+    boolean collectLearningRates();
+
+    //--- Histograms ---
+
+    /**
+     * Should histograms (per parameter type, or per layer for activations) of the given type be collected?
+     *
+     * @param type Stats type: Parameters, Updates, Activations
+     */
+    boolean collectHistograms(StatsType type);
+
+    /**
+     * Get the number of histogram bins to use for the given type (for use with {@link #collectHistograms(StatsType)}
+     *
+     * @param type Stats type: Parameters, Updates, Activatinos
+     */
+    int numHistogramBins(StatsType type);
+
+    //--- Summary Stats: Mean, Variance, Mean Magnitudes ---
+
+    /**
+     * Should the mean values (per parameter type, or per layer for activations) be collected?
+     *
+     * @param type Stats type: Parameters, Updates, Activations
+     */
+    boolean collectMean(StatsType type);
+
+    /**
+     * Should the standard devication values (per parameter type, or per layer for activations) be collected?
+     *
+     * @param type Stats type: Parameters, Updates, Activations
+     */
+    boolean collectStdev(StatsType type);
+
+    /**
+     * Should the mean magnitude values (per parameter type, or per layer for activations) be collected?
+     *
+     * @param type Stats type: Parameters, Updates, Activations
+     */
+    boolean collectMeanMagnitudes(StatsType type);
+
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/SummaryType.java

@@ -0,0 +1,27 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.api;
+
+import org.deeplearning4j.ui.model.stats.StatsListener;
+
+public enum SummaryType {
+    Mean, Stdev, MeanMagnitudes
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/DefaultStatsInitializationConfiguration.java

@@ -0,0 +1,47 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.impl;
+
+import lombok.AllArgsConstructor;
+import org.deeplearning4j.ui.model.stats.api.StatsInitializationConfiguration;
+
+@AllArgsConstructor
+public class DefaultStatsInitializationConfiguration implements StatsInitializationConfiguration {
+
+    private final boolean collectSoftwareInfo;
+    private final boolean collectHardwareInfo;
+    private final boolean collectModelInfo;
+
+    @Override
+    public boolean collectSoftwareInfo() {
+        return collectSoftwareInfo;
+    }
+
+    @Override
+    public boolean collectHardwareInfo() {
+        return collectHardwareInfo;
+    }
+
+    @Override
+    public boolean collectModelInfo() {
+        return collectModelInfo;
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/DefaultStatsUpdateConfiguration.java

@@ -0,0 +1,308 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.impl;
+
+import lombok.AllArgsConstructor;
+import org.deeplearning4j.ui.model.stats.api.StatsType;
+import org.deeplearning4j.ui.model.stats.api.StatsUpdateConfiguration;
+
+@AllArgsConstructor
+public class DefaultStatsUpdateConfiguration implements StatsUpdateConfiguration {
+
+    public static final int DEFAULT_REPORTING_FREQUENCY = 10;
+
+    private int reportingFrequency = DEFAULT_REPORTING_FREQUENCY;
+    private boolean collectPerformanceStats = true;
+    private boolean collectMemoryStats = true;
+    private boolean collectGarbageCollectionStats = true;
+    private boolean collectLearningRates = true;
+    private boolean collectHistogramsParameters = true;
+    private boolean collectHistogramsGradients = true;
+    private boolean collectHistogramsUpdates = true;
+    private boolean collectHistogramsActivations = true;
+    private int numHistogramBins = 20;
+    private boolean collectMeanParameters = true;
+    private boolean collectMeanGradients = true;
+    private boolean collectMeanUpdates = true;
+    private boolean collectMeanActivations = true;
+    private boolean collectStdevParameters = true;
+    private boolean collectStdevGradients = true;
+    private boolean collectStdevUpdates = true;
+    private boolean collectStdevActivations = true;
+    private boolean collectMeanMagnitudesParameters = true;
+    private boolean collectMeanMagnitudesGradients = true;
+    private boolean collectMeanMagnitudesUpdates = true;
+    private boolean collectMeanMagnitudesActivations = true;
+
+    private DefaultStatsUpdateConfiguration(Builder b) {
+        this.reportingFrequency = b.reportingFrequency;
+        this.collectPerformanceStats = b.collectPerformanceStats;
+        this.collectMemoryStats = b.collectMemoryStats;
+        this.collectGarbageCollectionStats = b.collectGarbageCollectionStats;
+        this.collectLearningRates = b.collectLearningRates;
+        this.collectHistogramsParameters = b.collectHistogramsParameters;
+        this.collectHistogramsGradients = b.collectHistogramsGradients;
+        this.collectHistogramsUpdates = b.collectHistogramsUpdates;
+        this.collectHistogramsActivations = b.collectHistogramsActivations;
+        this.numHistogramBins = b.numHistogramBins;
+        this.collectMeanParameters = b.collectMeanParameters;
+        this.collectMeanGradients = b.collectMeanGradients;
+        this.collectMeanUpdates = b.collectMeanUpdates;
+        this.collectMeanActivations = b.collectMeanActivations;
+        this.collectStdevParameters = b.collectStdevParameters;
+        this.collectStdevGradients = b.collectStdevGradients;
+        this.collectStdevUpdates = b.collectStdevUpdates;
+        this.collectStdevActivations = b.collectStdevActivations;
+        this.collectMeanMagnitudesParameters = b.collectMeanMagnitudesParameters;
+        this.collectMeanMagnitudesGradients = b.collectMeanMagnitudesGradients;
+        this.collectMeanMagnitudesUpdates = b.collectMeanMagnitudesUpdates;
+        this.collectMeanMagnitudesActivations = b.collectMeanMagnitudesActivations;
+    }
+
+    @Override
+    public int reportingFrequency() {
+        return reportingFrequency;
+    }
+
+    @Override
+    public boolean collectPerformanceStats() {
+        return collectPerformanceStats;
+    }
+
+    @Override
+    public boolean collectMemoryStats() {
+        return collectMemoryStats;
+    }
+
+    @Override
+    public boolean collectGarbageCollectionStats() {
+        return collectGarbageCollectionStats;
+    }
+
+    @Override
+    public boolean collectLearningRates() {
+        return collectLearningRates;
+    }
+
+    @Override
+    public boolean collectHistograms(StatsType type) {
+        switch (type) {
+            case Parameters:
+                return collectHistogramsParameters;
+            case Gradients:
+                return collectStdevGradients;
+            case Updates:
+                return collectHistogramsUpdates;
+            case Activations:
+                return collectHistogramsActivations;
+        }
+        return false;
+    }
+
+    @Override
+    public int numHistogramBins(StatsType type) {
+        return numHistogramBins;
+    }
+
+    @Override
+    public boolean collectMean(StatsType type) {
+        switch (type) {
+            case Parameters:
+                return collectMeanParameters;
+            case Gradients:
+                return collectMeanGradients;
+            case Updates:
+                return collectMeanUpdates;
+            case Activations:
+                return collectMeanActivations;
+        }
+        return false;
+    }
+
+    @Override
+    public boolean collectStdev(StatsType type) {
+        switch (type) {
+            case Parameters:
+                return collectStdevParameters;
+            case Gradients:
+                return collectStdevGradients;
+            case Updates:
+                return collectStdevUpdates;
+            case Activations:
+                return collectStdevActivations;
+        }
+        return false;
+    }
+
+    @Override
+    public boolean collectMeanMagnitudes(StatsType type) {
+        switch (type) {
+            case Parameters:
+                return collectMeanMagnitudesParameters;
+            case Gradients:
+                return collectMeanMagnitudesGradients;
+            case Updates:
+                return collectMeanMagnitudesUpdates;
+            case Activations:
+                return collectMeanMagnitudesActivations;
+        }
+        return false;
+    }
+
+    public static class Builder {
+        private int reportingFrequency = DEFAULT_REPORTING_FREQUENCY;
+        private boolean collectPerformanceStats = true;
+        private boolean collectMemoryStats = true;
+        private boolean collectGarbageCollectionStats = true;
+        private boolean collectLearningRates = true;
+        private boolean collectHistogramsParameters = true;
+        private boolean collectHistogramsGradients = true;
+        private boolean collectHistogramsUpdates = true;
+        private boolean collectHistogramsActivations = true;
+        private int numHistogramBins = 20;
+        private boolean collectMeanParameters = true;
+        private boolean collectMeanGradients = true;
+        private boolean collectMeanUpdates = true;
+        private boolean collectMeanActivations = true;
+        private boolean collectStdevParameters = true;
+        private boolean collectStdevGradients = true;
+        private boolean collectStdevUpdates = true;
+        private boolean collectStdevActivations = true;
+        private boolean collectMeanMagnitudesParameters = true;
+        private boolean collectMeanMagnitudesGradients = true;
+        private boolean collectMeanMagnitudesUpdates = true;
+        private boolean collectMeanMagnitudesActivations = true;
+
+        public Builder reportingFrequency(int reportingFrequency) {
+            this.reportingFrequency = reportingFrequency;
+            return this;
+        }
+
+        public Builder collectPerformanceStats(boolean collectPerformanceStats) {
+            this.collectPerformanceStats = collectPerformanceStats;
+            return this;
+        }
+
+        public Builder collectMemoryStats(boolean collectMemoryStats) {
+            this.collectMemoryStats = collectMemoryStats;
+            return this;
+        }
+
+        public Builder collectGarbageCollectionStats(boolean collectGarbageCollectionStats) {
+            this.collectGarbageCollectionStats = collectGarbageCollectionStats;
+            return this;
+        }
+
+        public Builder collectLearningRates(boolean collectLearningRates) {
+            this.collectLearningRates = collectLearningRates;
+            return this;
+        }
+
+        public Builder collectHistogramsParameters(boolean collectHistogramsParameters) {
+            this.collectHistogramsParameters = collectHistogramsParameters;
+            return this;
+        }
+
+        public Builder collectHistogramsGradients(boolean collectHistogramsGradients) {
+            this.collectHistogramsGradients = collectHistogramsGradients;
+            return this;
+        }
+
+        public Builder collectHistogramsUpdates(boolean collectHistogramsUpdates) {
+            this.collectHistogramsUpdates = collectHistogramsUpdates;
+            return this;
+        }
+
+        public Builder collectHistogramsActivations(boolean isCollectHistogramsActivations) {
+            this.collectHistogramsActivations = isCollectHistogramsActivations;
+            return this;
+        }
+
+        public Builder numHistogramBins(int numHistogramBins) {
+            this.numHistogramBins = numHistogramBins;
+            return this;
+        }
+
+        public Builder collectMeanParameters(boolean collectMeanParameters) {
+            this.collectMeanParameters = collectMeanParameters;
+            return this;
+        }
+
+        public Builder collectMeanGradients(boolean collectMeanGradients) {
+            this.collectMeanGradients = collectMeanGradients;
+            return this;
+        }
+
+        public Builder collectMeanUpdates(boolean collectMeanUpdates) {
+            this.collectMeanUpdates = collectMeanUpdates;
+            return this;
+        }
+
+        public Builder collectMeanActivations(boolean collectMeanActivations) {
+            this.collectMeanActivations = collectMeanActivations;
+            return this;
+        }
+
+        public Builder collectStdevParameters(boolean collectStdevParameters) {
+            this.collectStdevParameters = collectStdevParameters;
+            return this;
+        }
+
+        public Builder collectStdevGradients(boolean collectStdevGradients) {
+            this.collectStdevGradients = collectStdevGradients;
+            return this;
+        }
+
+        public Builder collectStdevUpdates(boolean collectStdevUpdates) {
+            this.collectStdevUpdates = collectStdevUpdates;
+            return this;
+        }
+
+        public Builder collectStdevActivations(boolean collectStdevActivations) {
+            this.collectStdevActivations = collectStdevActivations;
+            return this;
+        }
+
+        public Builder collectMeanMagnitudesParameters(boolean collectMeanMagnitudesParameters) {
+            this.collectMeanMagnitudesParameters = collectMeanMagnitudesParameters;
+            return this;
+        }
+
+        public Builder collectMeanMagnitudesGradients(boolean collectMeanMagnitudesGradients) {
+            this.collectMeanMagnitudesGradients = collectMeanMagnitudesGradients;
+            return this;
+        }
+
+        public Builder collectMeanMagnitudesUpdates(boolean collectMeanMagnitudesUpdates) {
+            this.collectMeanMagnitudesUpdates = collectMeanMagnitudesUpdates;
+            return this;
+        }
+
+        public Builder collectMeanMagnitudesActivations(boolean collectMeanMagnitudesActivations) {
+            this.collectMeanMagnitudesActivations = collectMeanMagnitudesActivations;
+            return this;
+        }
+
+        public DefaultStatsUpdateConfiguration build() {
+            return new DefaultStatsUpdateConfiguration(this);
+        }
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/SbeStatsInitializationReport.java

@@ -0,0 +1,468 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.impl;
+
+import lombok.Data;
+
+import org.agrona.DirectBuffer;
+import org.agrona.MutableDirectBuffer;
+import org.agrona.concurrent.UnsafeBuffer;
+import org.apache.commons.io.IOUtils;
+import org.deeplearning4j.ui.model.stats.api.StatsInitializationReport;
+import org.deeplearning4j.ui.model.stats.sbe.*;
+import org.deeplearning4j.ui.model.storage.AgronaPersistable;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+@Data
+public class SbeStatsInitializationReport implements StatsInitializationReport, AgronaPersistable {
+
+    private String sessionID;
+    private String typeID;
+    private String workerID;
+    private long timeStamp;
+
+    private boolean hasSoftwareInfo;
+    private boolean hasHardwareInfo;
+    private boolean hasModelInfo;
+
+    private String swArch;
+    private String swOsName;
+    private String swJvmName;
+    private String swJvmVersion;
+    private String swJvmSpecVersion;
+    private String swNd4jBackendClass;
+    private String swNd4jDataTypeName;
+    private String swHostName;
+    private String swJvmUID;
+    private Map<String, String> swEnvironmentInfo;
+
+    private int hwJvmAvailableProcessors;
+    private int hwNumDevices;
+    private long hwJvmMaxMemory;
+    private long hwOffHeapMaxMemory;
+    private long[] hwDeviceTotalMemory;
+    private String[] hwDeviceDescription;
+    private String hwHardwareUID;
+
+    private String modelClassName;
+    private String modelConfigJson;
+    private String[] modelParamNames;
+    private int modelNumLayers;
+    private long modelNumParams;
+
+    @Override
+    public void reportIDs(String sessionID, String typeID, String workerID, long timeStamp) {
+        this.sessionID = sessionID;
+        this.typeID = typeID;
+        this.workerID = workerID;
+        this.timeStamp = timeStamp;
+    }
+
+    @Override
+    public void reportSoftwareInfo(String arch, String osName, String jvmName, String jvmVersion, String jvmSpecVersion,
+                    String nd4jBackendClass, String nd4jDataTypeName, String hostname, String jvmUid,
+                    Map<String, String> swEnvironmentInfo) {
+        this.swArch = arch;
+        this.swOsName = osName;
+        this.swJvmName = jvmName;
+        this.swJvmVersion = jvmVersion;
+        this.swJvmSpecVersion = jvmSpecVersion;
+        this.swNd4jBackendClass = nd4jBackendClass;
+        this.swNd4jDataTypeName = nd4jDataTypeName;
+        this.swHostName = hostname;
+        this.swJvmUID = jvmUid;
+        this.swEnvironmentInfo = swEnvironmentInfo;
+        hasSoftwareInfo = true;
+    }
+
+    @Override
+    public void reportHardwareInfo(int jvmAvailableProcessors, int numDevices, long jvmMaxMemory, long offHeapMaxMemory,
+                    long[] deviceTotalMemory, String[] deviceDescription, String hardwareUID) {
+        this.hwJvmAvailableProcessors = jvmAvailableProcessors;
+        this.hwNumDevices = numDevices;
+        this.hwJvmMaxMemory = jvmMaxMemory;
+        this.hwOffHeapMaxMemory = offHeapMaxMemory;
+        this.hwDeviceTotalMemory = deviceTotalMemory;
+        this.hwDeviceDescription = deviceDescription;
+        this.hwHardwareUID = hardwareUID;
+        hasHardwareInfo = true;
+    }
+
+    @Override
+    public void reportModelInfo(String modelClassName, String modelConfigJson, String[] modelParamNames, int numLayers,
+                    long numParams) {
+        this.modelClassName = modelClassName;
+        this.modelConfigJson = modelConfigJson;
+        this.modelParamNames = modelParamNames;
+        this.modelNumLayers = numLayers;
+        this.modelNumParams = numParams;
+        hasModelInfo = true;
+    }
+
+    @Override
+    public boolean hasSoftwareInfo() {
+        return hasSoftwareInfo;
+    }
+
+    @Override
+    public boolean hasHardwareInfo() {
+        return hasHardwareInfo;
+    }
+
+    @Override
+    public boolean hasModelInfo() {
+        return hasModelInfo;
+    }
+
+
+
+    private void clearHwFields() {
+        hwDeviceTotalMemory = null;
+        hwDeviceDescription = null;
+        hwHardwareUID = null;
+    }
+
+    private void clearSwFields() {
+        swArch = null;
+        swOsName = null;
+        swJvmName = null;
+        swJvmVersion = null;
+        swJvmSpecVersion = null;
+        swNd4jBackendClass = null;
+        swNd4jDataTypeName = null;
+        swHostName = null;
+        swJvmUID = null;
+    }
+
+    private void clearModelFields() {
+        modelClassName = null;
+        modelConfigJson = null;
+        modelParamNames = null;
+    }
+
+    @Override
+    public String getSessionID() {
+        return sessionID;
+    }
+
+    @Override
+    public String getTypeID() {
+        return typeID;
+    }
+
+    @Override
+    public String getWorkerID() {
+        return workerID;
+    }
+
+    @Override
+    public long getTimeStamp() {
+        return timeStamp;
+    }
+
+    @Override
+    public int encodingLengthBytes() {
+        //TODO reuse the byte[]s here, to avoid converting them twice...
+
+        //First: need to determine how large a buffer to use.
+        //Buffer is composed of:
+        //(a) Header: 8 bytes (4x uint16 = 8 bytes)
+        //(b) Fixed length entries length (sie.BlockLength())
+        //(c) Group 1: Hardware devices (GPUs) max memory: 4 bytes header + nEntries * 8 (int64) + nEntries * variable length Strings (header + content)  = 4 + 8*n + content
+        //(d) Group 2: Software device info: 4 bytes header + 2x variable length Strings for each
+        //(d) Group 3: Parameter names: 4 bytes header + nEntries * variable length strings (header + content) = 4 + content
+        //(e) Variable length fields: 15 String length fields. Size: 4 bytes header, plus content. 60 bytes header
+        //Fixed length + repeating groups + variable length...
+        StaticInfoEncoder sie = new StaticInfoEncoder();
+        int bufferSize = 8 + sie.sbeBlockLength() + 4 + 4 + 60; //header + fixed values + group headers + variable length headers
+
+        //For variable length field lengths: easist way is simply to convert to UTF-8
+        //Of course, it is possible to calculate it first - but we might as well convert (1 pass), rather than count then convert (2 passes)
+        byte[] bSessionId = SbeUtil.toBytes(true, sessionID);
+        byte[] bTypeId = SbeUtil.toBytes(true, typeID);
+        byte[] bWorkerId = SbeUtil.toBytes(true, workerID);
+
+        byte[] bswArch = SbeUtil.toBytes(hasSoftwareInfo, swArch);
+        byte[] bswOsName = SbeUtil.toBytes(hasSoftwareInfo, swOsName);
+        byte[] bswJvmName = SbeUtil.toBytes(hasSoftwareInfo, swJvmName);
+        byte[] bswJvmVersion = SbeUtil.toBytes(hasSoftwareInfo, swJvmVersion);
+        byte[] bswJvmSpecVersion = SbeUtil.toBytes(hasSoftwareInfo, swJvmSpecVersion);
+        byte[] bswNd4jBackendClass = SbeUtil.toBytes(hasSoftwareInfo, swNd4jBackendClass);
+        byte[] bswNd4jDataTypeName = SbeUtil.toBytes(hasSoftwareInfo, swNd4jDataTypeName);
+        byte[] bswHostname = SbeUtil.toBytes(hasSoftwareInfo, swHostName);
+        byte[] bswJvmUID = SbeUtil.toBytes(hasSoftwareInfo, swJvmUID);
+        byte[] bHwHardwareUID = SbeUtil.toBytes(hasHardwareInfo, hwHardwareUID);
+        byte[] bmodelConfigClass = SbeUtil.toBytes(hasModelInfo, modelClassName);
+        byte[] bmodelConfigJson = SbeUtil.toBytes(hasModelInfo, modelConfigJson);
+
+        byte[][] bhwDeviceDescription = SbeUtil.toBytes(hasHardwareInfo, hwDeviceDescription);
+        byte[][][] bswEnvInfo = SbeUtil.toBytes(swEnvironmentInfo);
+        byte[][] bModelParamNames = SbeUtil.toBytes(hasModelInfo, modelParamNames);
+
+
+
+        bufferSize += bSessionId.length + bTypeId.length + bWorkerId.length;
+
+        bufferSize += 4; //swEnvironmentInfo group header (always present)
+        if (hasSoftwareInfo) {
+            bufferSize += SbeUtil.length(bswArch);
+            bufferSize += SbeUtil.length(bswOsName);
+            bufferSize += SbeUtil.length(bswJvmName);
+            bufferSize += SbeUtil.length(bswJvmVersion);
+            bufferSize += SbeUtil.length(bswJvmSpecVersion);
+            bufferSize += SbeUtil.length(bswNd4jBackendClass);
+            bufferSize += SbeUtil.length(bswNd4jDataTypeName);
+            bufferSize += SbeUtil.length(bswHostname);
+            bufferSize += SbeUtil.length(bswJvmUID);
+            //For each entry: 2 variable-length headers (2x4 bytes each) + content
+            int envCount = (bswEnvInfo != null ? bswEnvInfo.length : 0);
+            bufferSize += envCount * 8;
+            bufferSize += SbeUtil.length(bswEnvInfo);
+        }
+        int nHWDeviceStats = hwNumDevices;
+        if (!hasHardwareInfo)
+            nHWDeviceStats = 0;
+        if (hasHardwareInfo) {
+            //Device info group:
+            bufferSize += hwNumDevices * 8; //fixed content in group: int64 -> 8 bytes. Encode an entry, even if hwDeviceTotalMemory is null
+            bufferSize += hwNumDevices * 4; //uint32: 4 bytes per entry for var length header...; as above
+            bufferSize += SbeUtil.length(bhwDeviceDescription);
+            bufferSize += SbeUtil.length(bHwHardwareUID);
+        }
+        if (hasModelInfo) {
+            bufferSize += SbeUtil.length(bmodelConfigClass);
+            bufferSize += SbeUtil.length(bmodelConfigJson);
+            bufferSize += SbeUtil.length(bModelParamNames);
+            bufferSize += (bModelParamNames == null ? 0 : bModelParamNames.length * 4); //uint32: 4 bytes per entry for var length header...
+        }
+
+        return bufferSize;
+    }
+
+    @Override
+    public byte[] encode() {
+        byte[] bytes = new byte[encodingLengthBytes()];
+        MutableDirectBuffer buffer = new UnsafeBuffer(bytes);
+        encode(buffer);
+        return bytes;
+    }
+
+    @Override
+    public void encode(ByteBuffer buffer) {
+        encode(new UnsafeBuffer(buffer));
+    }
+
+    @Override
+    public void encode(MutableDirectBuffer buffer) {
+
+        MessageHeaderEncoder enc = new MessageHeaderEncoder();
+        StaticInfoEncoder sie = new StaticInfoEncoder();
+
+        byte[] bSessionId = SbeUtil.toBytes(true, sessionID);
+        byte[] bTypeId = SbeUtil.toBytes(true, typeID);
+        byte[] bWorkerId = SbeUtil.toBytes(true, workerID);
+
+        byte[] bswArch = SbeUtil.toBytes(hasSoftwareInfo, swArch);
+        byte[] bswOsName = SbeUtil.toBytes(hasSoftwareInfo, swOsName);
+        byte[] bswJvmName = SbeUtil.toBytes(hasSoftwareInfo, swJvmName);
+        byte[] bswJvmVersion = SbeUtil.toBytes(hasSoftwareInfo, swJvmVersion);
+        byte[] bswJvmSpecVersion = SbeUtil.toBytes(hasSoftwareInfo, swJvmSpecVersion);
+        byte[] bswNd4jBackendClass = SbeUtil.toBytes(hasSoftwareInfo, swNd4jBackendClass);
+        byte[] bswNd4jDataTypeName = SbeUtil.toBytes(hasSoftwareInfo, swNd4jDataTypeName);
+        byte[] bswHostname = SbeUtil.toBytes(hasSoftwareInfo, swHostName);
+        byte[] bswJvmUID = SbeUtil.toBytes(hasSoftwareInfo, swJvmUID);
+        byte[] bHwHardwareUID = SbeUtil.toBytes(hasHardwareInfo, hwHardwareUID);
+        byte[] bmodelConfigClass = SbeUtil.toBytes(hasModelInfo, modelClassName);
+        byte[] bmodelConfigJson = SbeUtil.toBytes(hasModelInfo, modelConfigJson);
+
+        byte[][] bhwDeviceDescription = SbeUtil.toBytes(hasHardwareInfo, hwDeviceDescription);
+        byte[][][] bswEnvInfo = SbeUtil.toBytes(swEnvironmentInfo);
+        byte[][] bModelParamNames = SbeUtil.toBytes(hasModelInfo, modelParamNames);
+
+        enc.wrap(buffer, 0).blockLength(sie.sbeBlockLength()).templateId(sie.sbeTemplateId())
+                        .schemaId(sie.sbeSchemaId()).version(sie.sbeSchemaVersion());
+
+        int offset = enc.encodedLength(); //Expect 8 bytes...
+
+        //Fixed length fields: always encoded, whether present or not.
+        sie.wrap(buffer, offset).time(timeStamp).fieldsPresent().softwareInfo(hasSoftwareInfo)
+                        .hardwareInfo(hasHardwareInfo).modelInfo(hasModelInfo);
+        sie.hwJvmProcessors(hwJvmAvailableProcessors).hwNumDevices((short) hwNumDevices).hwJvmMaxMemory(hwJvmMaxMemory)
+                        .hwOffheapMaxMemory(hwOffHeapMaxMemory).modelNumLayers(modelNumLayers)
+                        .modelNumParams(modelNumParams);
+        //Device info group...
+        StaticInfoEncoder.HwDeviceInfoGroupEncoder hwdEnc = sie.hwDeviceInfoGroupCount(hwNumDevices);
+        int nHWDeviceStats = (hasHardwareInfo ? hwNumDevices : 0);
+        for (int i = 0; i < nHWDeviceStats; i++) {
+            long maxMem = hwDeviceTotalMemory == null || hwDeviceTotalMemory.length <= i ? 0 : hwDeviceTotalMemory[i];
+            byte[] descr = bhwDeviceDescription == null || bhwDeviceDescription.length <= i ? SbeUtil.EMPTY_BYTES
+                            : bhwDeviceDescription[i];
+            if (descr == null)
+                descr = SbeUtil.EMPTY_BYTES;
+            hwdEnc.next().deviceMemoryMax(maxMem).putDeviceDescription(descr, 0, descr.length);
+        }
+
+        //Environment info group
+        int numEnvValues = (hasSoftwareInfo && swEnvironmentInfo != null ? swEnvironmentInfo.size() : 0);
+        StaticInfoEncoder.SwEnvironmentInfoEncoder swEnv = sie.swEnvironmentInfoCount(numEnvValues);
+        if (numEnvValues > 0) {
+            byte[][][] mapAsBytes = SbeUtil.toBytes(swEnvironmentInfo);
+            for (byte[][] entryBytes : mapAsBytes) {
+                swEnv.next().putEnvKey(entryBytes[0], 0, entryBytes[0].length).putEnvValue(entryBytes[1], 0,
+                                entryBytes[1].length);
+            }
+        }
+
+        int nParamNames = modelParamNames == null ? 0 : modelParamNames.length;
+        StaticInfoEncoder.ModelParamNamesEncoder mpnEnc = sie.modelParamNamesCount(nParamNames);
+        for (int i = 0; i < nParamNames; i++) {
+            mpnEnc.next().putModelParamNames(bModelParamNames[i], 0, bModelParamNames[i].length);
+        }
+
+        //In the case of !hasSoftwareInfo: these will all be empty byte arrays... still need to encode them (for 0 length) however
+        sie.putSessionID(bSessionId, 0, bSessionId.length).putTypeID(bTypeId, 0, bTypeId.length)
+                        .putWorkerID(bWorkerId, 0, bWorkerId.length).putSwArch(bswArch, 0, bswArch.length)
+                        .putSwOsName(bswOsName, 0, bswOsName.length).putSwJvmName(bswJvmName, 0, bswJvmName.length)
+                        .putSwJvmVersion(bswJvmVersion, 0, bswJvmVersion.length)
+                        .putSwJvmSpecVersion(bswJvmSpecVersion, 0, bswJvmSpecVersion.length)
+                        .putSwNd4jBackendClass(bswNd4jBackendClass, 0, bswNd4jBackendClass.length)
+                        .putSwNd4jDataTypeName(bswNd4jDataTypeName, 0, bswNd4jDataTypeName.length)
+                        .putSwHostName(bswHostname, 0, bswHostname.length).putSwJvmUID(bswJvmUID, 0, bswJvmUID.length)
+                        .putHwHardwareUID(bHwHardwareUID, 0, bHwHardwareUID.length);
+        //Similar: !hasModelInfo -> empty byte[]
+        sie.putModelConfigClassName(bmodelConfigClass, 0, bmodelConfigClass.length).putModelConfigJson(bmodelConfigJson,
+                        0, bmodelConfigJson.length);
+    }
+
+    @Override
+    public void encode(OutputStream outputStream) throws IOException {
+        //TODO there may be more efficient way of doing this
+        outputStream.write(encode());
+    }
+
+    @Override
+    public void decode(byte[] decode) {
+        MutableDirectBuffer buffer = new UnsafeBuffer(decode);
+        decode(buffer);
+    }
+
+    @Override
+    public void decode(ByteBuffer buffer) {
+        decode(new UnsafeBuffer(buffer));
+    }
+
+    @Override
+    public void decode(DirectBuffer buffer) {
+        //TODO we could do this much more efficiently, with buffer re-use, etc.
+        MessageHeaderDecoder dec = new MessageHeaderDecoder();
+        StaticInfoDecoder sid = new StaticInfoDecoder();
+        dec.wrap(buffer, 0);
+
+        final int blockLength = dec.blockLength();
+        final int version = dec.version();
+
+        final int headerLength = dec.encodedLength();
+        //TODO: in general, we should check the header, version, schema etc. But we don't have any other versions yet.
+
+        sid.wrap(buffer, headerLength, blockLength, version);
+        timeStamp = sid.time();
+        InitFieldsPresentDecoder fields = sid.fieldsPresent();
+        hasSoftwareInfo = fields.softwareInfo();
+        hasHardwareInfo = fields.hardwareInfo();
+        hasModelInfo = fields.modelInfo();
+
+        //These fields: always present, even if !hasHardwareInfo
+        hwJvmAvailableProcessors = sid.hwJvmProcessors();
+        hwNumDevices = sid.hwNumDevices();
+        hwJvmMaxMemory = sid.hwJvmMaxMemory();
+        hwOffHeapMaxMemory = sid.hwOffheapMaxMemory();
+        modelNumLayers = sid.modelNumLayers();
+        modelNumParams = sid.modelNumParams();
+
+        //Hardware device info group
+        StaticInfoDecoder.HwDeviceInfoGroupDecoder hwDeviceInfoGroupDecoder = sid.hwDeviceInfoGroup();
+        int count = hwDeviceInfoGroupDecoder.count();
+        if (count > 0) {
+            hwDeviceTotalMemory = new long[count];
+            hwDeviceDescription = new String[count];
+        }
+        int i = 0;
+        for (StaticInfoDecoder.HwDeviceInfoGroupDecoder hw : hwDeviceInfoGroupDecoder) {
+            hwDeviceTotalMemory[i] = hw.deviceMemoryMax();
+            hwDeviceDescription[i++] = hw.deviceDescription();
+        }
+
+        //Environment info group
+        i = 0;
+        StaticInfoDecoder.SwEnvironmentInfoDecoder swEnvDecoder = sid.swEnvironmentInfo();
+        if (swEnvDecoder.count() > 0) {
+            swEnvironmentInfo = new HashMap<>();
+        }
+        for (StaticInfoDecoder.SwEnvironmentInfoDecoder env : swEnvDecoder) {
+            String key = env.envKey();
+            String value = env.envValue();
+            swEnvironmentInfo.put(key, value);
+        }
+
+        i = 0;
+        StaticInfoDecoder.ModelParamNamesDecoder mpdec = sid.modelParamNames();
+        int mpnCount = mpdec.count();
+        modelParamNames = new String[mpnCount];
+        for (StaticInfoDecoder.ModelParamNamesDecoder mp : mpdec) {
+            modelParamNames[i++] = mp.modelParamNames();
+        }
+        //Variable length data. Even if it is missing: still needs to be read, to advance buffer
+        //Again, the exact order of these calls matters here
+        sessionID = sid.sessionID();
+        typeID = sid.typeID();
+        workerID = sid.workerID();
+        swArch = sid.swArch();
+        swOsName = sid.swOsName();
+        swJvmName = sid.swJvmName();
+        swJvmVersion = sid.swJvmVersion();
+        swJvmSpecVersion = sid.swJvmSpecVersion();
+        swNd4jBackendClass = sid.swNd4jBackendClass();
+        swNd4jDataTypeName = sid.swNd4jDataTypeName();
+        swHostName = sid.swHostName();
+        swJvmUID = sid.swJvmUID();
+        if (!hasSoftwareInfo)
+            clearSwFields();
+        hwHardwareUID = sid.hwHardwareUID();
+        if (!hasHardwareInfo)
+            clearHwFields();
+        modelClassName = sid.modelConfigClassName();
+        modelConfigJson = sid.modelConfigJson();
+        if (!hasModelInfo)
+            clearModelFields();
+    }
+
+    @Override
+    public void decode(InputStream inputStream) throws IOException {
+        byte[] bytes = IOUtils.toByteArray(inputStream);
+        decode(bytes);
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/SbeUtil.java

@@ -0,0 +1,131 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.impl;
+
+import java.io.*;
+import java.nio.charset.Charset;
+import java.util.Map;
+
+public class SbeUtil {
+
+    public static final Charset UTF8 = Charset.forName("UTF-8");
+    public static final byte[] EMPTY_BYTES = new byte[0]; //Also equivalent to "".getBytes(UTF8);
+
+    private SbeUtil() {}
+
+    public static int length(byte[] bytes) {
+        if (bytes == null)
+            return 0;
+        return bytes.length;
+    }
+
+    public static int length(byte[][] bytes) {
+        if (bytes == null)
+            return 0;
+        int count = 0;
+        for (int i = 0; i < bytes.length; i++) {
+            if (bytes[i] != null)
+                count += bytes[i].length;
+        }
+        return count;
+    }
+
+    public static int length(byte[][][] bytes) {
+        if (bytes == null)
+            return 0;
+        int count = 0;
+        for (byte[][] arr : bytes) {
+            count += length(arr);
+        }
+        return count;
+    }
+
+    public static int length(String str) {
+        if (str == null)
+            return 0;
+        return str.length();
+    }
+
+    public static int length(String[] arr) {
+        if (arr == null || arr.length == 0)
+            return 0;
+        int sum = 0;
+        for (String s : arr)
+            sum += length(s);
+        return sum;
+    }
+
+    public static byte[] toBytes(boolean present, String str) {
+        if (!present || str == null)
+            return EMPTY_BYTES;
+        return str.getBytes(UTF8);
+    }
+
+    public static byte[][] toBytes(boolean present, String[] str) {
+        if (str == null)
+            return null;
+        byte[][] b = new byte[str.length][0];
+        for (int i = 0; i < str.length; i++) {
+            if (str[i] == null)
+                continue;
+            b[i] = toBytes(present, str[i]);
+        }
+        return b;
+    }
+
+    public static byte[][][] toBytes(Map<String, String> map) {
+        if (map == null)
+            return null;
+        byte[][][] b = new byte[map.size()][2][0];
+        int i = 0;
+        for (Map.Entry<String, String> entry : map.entrySet()) {
+            b[i][0] = toBytes(true, entry.getKey());
+            b[i][1] = toBytes(true, entry.getValue());
+            i++;
+        }
+        return b;
+    }
+
+    public static byte[] toBytesSerializable(Serializable serializable) {
+        if (serializable == null)
+            return EMPTY_BYTES;
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
+            oos.writeObject(serializable);
+        } catch (IOException e) {
+            throw new RuntimeException("Unexpected IOException during serialization", e);
+        }
+        return baos.toByteArray();
+    }
+
+    public static Serializable fromBytesSerializable(byte[] bytes) {
+        if (bytes == null || bytes.length == 0)
+            return null;
+        ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+        try (ObjectInputStream ois = new ObjectInputStream(bais)) {
+            return (Serializable) ois.readObject();
+        } catch (IOException e) {
+            throw new RuntimeException("Unexpected IOException during deserialization", e);
+        } catch (ClassNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/java/JavaStatsInitializationReport.java

@@ -0,0 +1,195 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.impl.java;
+
+import lombok.Data;
+import org.apache.commons.compress.utils.IOUtils;
+import org.deeplearning4j.ui.model.stats.api.StatsInitializationReport;
+
+import java.io.*;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+@Data
+public class JavaStatsInitializationReport implements StatsInitializationReport {
+
+    private String sessionID;
+    private String typeID;
+    private String workerID;
+    private long timeStamp;
+
+    private boolean hasSoftwareInfo;
+    private boolean hasHardwareInfo;
+    private boolean hasModelInfo;
+
+    private String swArch;
+    private String swOsName;
+    private String swJvmName;
+    private String swJvmVersion;
+    private String swJvmSpecVersion;
+    private String swNd4jBackendClass;
+    private String swNd4jDataTypeName;
+    private String swHostName;
+    private String swJvmUID;
+    private Map<String, String> swEnvironmentInfo;
+
+    private int hwJvmAvailableProcessors;
+    private int hwNumDevices;
+    private long hwJvmMaxMemory;
+    private long hwOffHeapMaxMemory;
+    private long[] hwDeviceTotalMemory;
+    private String[] hwDeviceDescription;
+    private String hwHardwareUID;
+
+    private String modelClassName;
+    private String modelConfigJson;
+    private String[] modelParamNames;
+    private int modelNumLayers;
+    private long modelNumParams;
+
+    @Override
+    public void reportIDs(String sessionID, String typeID, String workerID, long timeStamp) {
+        this.sessionID = sessionID;
+        this.typeID = typeID;
+        this.workerID = workerID;
+        this.timeStamp = timeStamp;
+    }
+
+    @Override
+    public void reportSoftwareInfo(String arch, String osName, String jvmName, String jvmVersion, String jvmSpecVersion,
+                    String nd4jBackendClass, String nd4jDataTypeName, String hostname, String jvmUid,
+                    Map<String, String> swEnvironmentInfo) {
+        this.swArch = arch;
+        this.swOsName = osName;
+        this.swJvmName = jvmName;
+        this.swJvmVersion = jvmVersion;
+        this.swJvmSpecVersion = jvmSpecVersion;
+        this.swNd4jBackendClass = nd4jBackendClass;
+        this.swNd4jDataTypeName = nd4jDataTypeName;
+        this.swHostName = hostname;
+        this.swJvmUID = jvmUid;
+        this.swEnvironmentInfo = swEnvironmentInfo;
+        hasSoftwareInfo = true;
+    }
+
+    @Override
+    public void reportHardwareInfo(int jvmAvailableProcessors, int numDevices, long jvmMaxMemory, long offHeapMaxMemory,
+                    long[] deviceTotalMemory, String[] deviceDescription, String hardwareUID) {
+        this.hwJvmAvailableProcessors = jvmAvailableProcessors;
+        this.hwNumDevices = numDevices;
+        this.hwJvmMaxMemory = jvmMaxMemory;
+        this.hwOffHeapMaxMemory = offHeapMaxMemory;
+        this.hwDeviceTotalMemory = deviceTotalMemory;
+        this.hwDeviceDescription = deviceDescription;
+        this.hwHardwareUID = hardwareUID;
+        hasHardwareInfo = true;
+    }
+
+    @Override
+    public void reportModelInfo(String modelClassName, String modelConfigJson, String[] modelParamNames, int numLayers,
+                    long numParams) {
+        this.modelClassName = modelClassName;
+        this.modelConfigJson = modelConfigJson;
+        this.modelParamNames = modelParamNames;
+        this.modelNumLayers = numLayers;
+        this.modelNumParams = numParams;
+        hasModelInfo = true;
+    }
+
+    @Override
+    public boolean hasSoftwareInfo() {
+        return hasSoftwareInfo;
+    }
+
+    @Override
+    public boolean hasHardwareInfo() {
+        return hasHardwareInfo;
+    }
+
+    @Override
+    public boolean hasModelInfo() {
+        return hasModelInfo;
+    }
+
+
+    @Override
+    public int encodingLengthBytes() {
+        //TODO - presumably a more efficient way to do this
+        byte[] encoded = encode();
+        return encoded.length;
+    }
+
+    @Override
+    public byte[] encode() {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
+            oos.writeObject(this);
+        } catch (IOException e) {
+            throw new RuntimeException(e); //Should never happen
+        }
+        return baos.toByteArray();
+    }
+
+    @Override
+    public void encode(ByteBuffer buffer) {
+        buffer.put(encode());
+    }
+
+    @Override
+    public void encode(OutputStream outputStream) throws IOException {
+        try (ObjectOutputStream oos = new ObjectOutputStream(outputStream)) {
+            oos.writeObject(this);
+        }
+    }
+
+    @Override
+    public void decode(byte[] decode) {
+        JavaStatsInitializationReport r;
+        try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(decode))) {
+            r = (JavaStatsInitializationReport) ois.readObject();
+        } catch (IOException | ClassNotFoundException e) {
+            throw new RuntimeException(e); //Should never happen
+        }
+
+        Field[] fields = JavaStatsInitializationReport.class.getDeclaredFields();
+        for (Field f : fields) {
+            f.setAccessible(true);
+            try {
+                f.set(this, f.get(r));
+            } catch (IllegalAccessException e) {
+                throw new RuntimeException(e); //Should never happen
+            }
+        }
+    }
+
+    @Override
+    public void decode(ByteBuffer buffer) {
+        byte[] bytes = new byte[buffer.remaining()];
+        buffer.get(bytes);
+        decode(bytes);
+    }
+
+    @Override
+    public void decode(InputStream inputStream) throws IOException {
+        decode(IOUtils.toByteArray(inputStream));
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/java/JavaStatsReport.java

@@ -0,0 +1,390 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.impl.java;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+import org.apache.commons.compress.utils.IOUtils;
+import org.deeplearning4j.ui.model.stats.api.Histogram;
+import org.deeplearning4j.ui.model.stats.api.StatsReport;
+import org.deeplearning4j.ui.model.stats.api.StatsType;
+import org.deeplearning4j.ui.model.stats.api.SummaryType;
+import org.nd4j.common.primitives.Pair;
+
+import java.io.*;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@EqualsAndHashCode
+@ToString
+@Data
+public class JavaStatsReport implements StatsReport {
+
+    private String sessionID;
+    private String typeID;
+    private String workerID;
+    private long timeStamp;
+
+    private int iterationCount;
+    private int statsCollectionDurationMs;
+    private double score;
+
+    private long jvmCurrentBytes;
+    private long jvmMaxBytes;
+    private long offHeapCurrentBytes;
+    private long offHeapMaxBytes;
+    private long[] deviceCurrentBytes;
+    private long[] deviceMaxBytes;
+
+    private long totalRuntimeMs;
+    private long totalExamples;
+    private long totalMinibatches;
+    private double examplesPerSecond;
+    private double minibatchesPerSecond;
+
+    private List<GCStats> gcStats;
+
+    private Map<String, Double> learningRatesByParam;
+    private Map<StatsType, Map<String, Histogram>> histograms;
+    private Map<StatsType, Map<String, Double>> meanValues;
+    private Map<StatsType, Map<String, Double>> stdevValues;
+    private Map<StatsType, Map<String, Double>> meanMagnitudeValues;
+
+    private String metaDataClassName;
+    //Store in serialized form; deserialize iff required. Might save us some class not found (or, version) errors, if
+    // metadata is saved but is never used
+    private List<byte[]> dataSetMetaData;
+
+    private boolean scorePresent;
+    private boolean memoryUsePresent;
+    private boolean performanceStatsPresent;
+
+    public JavaStatsReport() {
+        //No-Arg constructor only for deserialization
+    }
+
+
+    @Override
+    public void reportIDs(String sessionID, String typeID, String workerID, long timeStamp) {
+        this.sessionID = sessionID;
+        this.typeID = typeID;
+        this.workerID = workerID;
+        this.timeStamp = timeStamp;
+    }
+
+    @Override
+    public void reportIterationCount(int iterationCount) {
+        this.iterationCount = iterationCount;
+    }
+
+
+    @Override
+    public void reportStatsCollectionDurationMS(int statsCollectionDurationMS) {
+        this.statsCollectionDurationMs = statsCollectionDurationMS;
+    }
+
+    @Override
+    public void reportScore(double currentScore) {
+        this.score = currentScore;
+        this.scorePresent = true;
+    }
+
+    @Override
+    public void reportLearningRates(Map<String, Double> learningRatesByParam) {
+        this.learningRatesByParam = learningRatesByParam;
+    }
+
+    @Override
+    public void reportMemoryUse(long jvmCurrentBytes, long jvmMaxBytes, long offHeapCurrentBytes, long offHeapMaxBytes,
+                    long[] deviceCurrentBytes, long[] deviceMaxBytes) {
+        this.jvmCurrentBytes = jvmCurrentBytes;
+        this.jvmMaxBytes = jvmMaxBytes;
+        this.offHeapCurrentBytes = offHeapCurrentBytes;
+        this.offHeapMaxBytes = offHeapMaxBytes;
+        this.deviceCurrentBytes = deviceCurrentBytes;
+        this.deviceMaxBytes = deviceMaxBytes;
+        this.memoryUsePresent = true;
+    }
+
+    @Override
+    public void reportPerformance(long totalRuntimeMs, long totalExamples, long totalMinibatches,
+                    double examplesPerSecond, double minibatchesPerSecond) {
+        this.totalRuntimeMs = totalRuntimeMs;
+        this.totalExamples = totalExamples;
+        this.totalMinibatches = totalMinibatches;
+        this.examplesPerSecond = examplesPerSecond;
+        this.minibatchesPerSecond = minibatchesPerSecond;
+        this.performanceStatsPresent = true;
+    }
+
+    @Override
+    public void reportGarbageCollection(String gcName, int deltaGCCount, int deltaGCTime) {
+        if (gcStats == null)
+            gcStats = new ArrayList<>();
+        gcStats.add(new GCStats(gcName, deltaGCCount, deltaGCTime));
+    }
+
+    @Override
+    public List<Pair<String, int[]>> getGarbageCollectionStats() {
+        if (gcStats == null)
+            return null;
+        List<Pair<String, int[]>> temp = new ArrayList<>();
+        for (GCStats g : gcStats) {
+            temp.add(new Pair<>(g.gcName, new int[] {g.getDeltaGCCount(), g.getDeltaGCTime()}));
+        }
+        return temp;
+    }
+
+    @Override
+    public void reportHistograms(StatsType statsType, Map<String, Histogram> histogram) {
+        if (this.histograms == null)
+            this.histograms = new HashMap<>();
+        this.histograms.put(statsType, histogram);
+    }
+
+    @Override
+    public Map<String, Histogram> getHistograms(StatsType statsType) {
+        if (histograms == null)
+            return null;
+        return histograms.get(statsType);
+    }
+
+    @Override
+    public void reportMean(StatsType statsType, Map<String, Double> mean) {
+        if (this.meanValues == null)
+            this.meanValues = new HashMap<>();
+        this.meanValues.put(statsType, mean);
+    }
+
+    @Override
+    public Map<String, Double> getMean(StatsType statsType) {
+        if (this.meanValues == null)
+            return null;
+        return meanValues.get(statsType);
+    }
+
+    @Override
+    public void reportStdev(StatsType statsType, Map<String, Double> stdev) {
+        if (this.stdevValues == null)
+            this.stdevValues = new HashMap<>();
+        this.stdevValues.put(statsType, stdev);
+    }
+
+    @Override
+    public Map<String, Double> getStdev(StatsType statsType) {
+        if (this.stdevValues == null)
+            return null;
+        return stdevValues.get(statsType);
+    }
+
+    @Override
+    public void reportMeanMagnitudes(StatsType statsType, Map<String, Double> meanMagnitudes) {
+        if (this.meanMagnitudeValues == null)
+            this.meanMagnitudeValues = new HashMap<>();
+        this.meanMagnitudeValues.put(statsType, meanMagnitudes);
+    }
+
+    @Override
+    public void reportDataSetMetaData(List<Serializable> dataSetMetaData, Class<?> metaDataClass) {
+        reportDataSetMetaData(dataSetMetaData, (metaDataClass == null ? null : metaDataClass.getName()));
+    }
+
+    @Override
+    public void reportDataSetMetaData(List<Serializable> dataSetMetaData, String metaDataClass) {
+        if (dataSetMetaData != null) {
+            this.dataSetMetaData = new ArrayList<>();
+            for (Serializable s : dataSetMetaData) {
+                ByteArrayOutputStream baos = new ByteArrayOutputStream();
+                try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
+                    oos.writeObject(s);
+                    oos.flush();
+                    oos.close();
+                } catch (IOException e) {
+                    throw new RuntimeException("Unexpected IOException from ByteArrayOutputStream", e);
+                }
+                byte[] b = baos.toByteArray();
+                this.dataSetMetaData.add(b);
+            }
+        } else {
+            this.dataSetMetaData = null;
+        }
+        this.metaDataClassName = metaDataClass;
+    }
+
+    @Override
+    public Map<String, Double> getMeanMagnitudes(StatsType statsType) {
+        if (this.meanMagnitudeValues == null)
+            return null;
+        return this.meanMagnitudeValues.get(statsType);
+    }
+
+    @Override
+    public List<Serializable> getDataSetMetaData() {
+        if (dataSetMetaData == null || dataSetMetaData.isEmpty())
+            return null;
+
+        List<Serializable> l = new ArrayList<>();
+        for (byte[] b : dataSetMetaData) {
+            try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(b))) {
+                l.add((Serializable) ois.readObject());
+            } catch (IOException | ClassNotFoundException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        return l;
+    }
+
+    @Override
+    public String getDataSetMetaDataClassName() {
+        return metaDataClassName;
+    }
+
+    @Override
+    public Map<String, Double> getLearningRates() {
+        return this.learningRatesByParam;
+    }
+
+    @Override
+    public boolean hasScore() {
+        return scorePresent;
+    }
+
+    @Override
+    public boolean hasLearningRates() {
+        return learningRatesByParam != null;
+    }
+
+    @Override
+    public boolean hasMemoryUse() {
+        return memoryUsePresent;
+    }
+
+    @Override
+    public boolean hasPerformance() {
+        return performanceStatsPresent;
+    }
+
+    @Override
+    public boolean hasGarbageCollection() {
+        return gcStats != null && !gcStats.isEmpty();
+    }
+
+    @Override
+    public boolean hasHistograms(StatsType statsType) {
+        if (histograms == null)
+            return false;
+        return histograms.containsKey(statsType);
+    }
+
+    @Override
+    public boolean hasSummaryStats(StatsType statsType, SummaryType summaryType) {
+        switch (summaryType) {
+            case Mean:
+                return meanValues != null && meanValues.containsKey(statsType);
+            case Stdev:
+                return stdevValues != null && stdevValues.containsKey(statsType);
+            case MeanMagnitudes:
+                return meanMagnitudeValues != null && meanMagnitudeValues.containsKey(statsType);
+        }
+        return false;
+    }
+
+    @Override
+    public boolean hasDataSetMetaData() {
+        return dataSetMetaData != null || metaDataClassName != null;
+    }
+
+    @AllArgsConstructor
+    @Data
+    private static class GCStats implements Serializable {
+        private String gcName;
+        private int deltaGCCount;
+        private int deltaGCTime;
+    }
+
+    @Override
+    public int encodingLengthBytes() {
+        //TODO - presumably a more efficient way to do this
+        byte[] encoded = encode();
+        return encoded.length;
+    }
+
+    @Override
+    public byte[] encode() {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
+            oos.writeObject(this);
+        } catch (IOException e) {
+            throw new RuntimeException(e); //Should never happen
+        }
+        return baos.toByteArray();
+    }
+
+    @Override
+    public void encode(ByteBuffer buffer) {
+        buffer.put(encode());
+    }
+
+    @Override
+    public void encode(OutputStream outputStream) throws IOException {
+        try (ObjectOutputStream oos = new ObjectOutputStream(outputStream)) {
+            oos.writeObject(this);
+        }
+    }
+
+    @Override
+    public void decode(byte[] decode) {
+        JavaStatsReport r;
+        try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(decode))) {
+            r = (JavaStatsReport) ois.readObject();
+        } catch (IOException | ClassNotFoundException e) {
+            throw new RuntimeException(e); //Should never happen
+        }
+
+        Field[] fields = JavaStatsReport.class.getDeclaredFields();
+        for (Field f : fields) {
+            f.setAccessible(true);
+            try {
+                f.set(this, f.get(r));
+            } catch (IllegalAccessException e) {
+                throw new RuntimeException(e); //Should never happen
+            }
+        }
+    }
+
+    @Override
+    public void decode(ByteBuffer buffer) {
+        byte[] bytes = new byte[buffer.remaining()];
+        buffer.get(bytes);
+        decode(bytes);
+    }
+
+    @Override
+    public void decode(InputStream inputStream) throws IOException {
+        decode(IOUtils.toByteArray(inputStream));
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/GroupSizeEncodingDecoder.java

@@ -0,0 +1,93 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+import org.agrona.DirectBuffer;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.GroupSizeEncodingDecoder"})
+@SuppressWarnings("all")
+public class GroupSizeEncodingDecoder {
+    public static final int ENCODED_LENGTH = 4;
+    private DirectBuffer buffer;
+    private int offset;
+
+    public GroupSizeEncodingDecoder wrap(final DirectBuffer buffer, final int offset) {
+        this.buffer = buffer;
+        this.offset = offset;
+
+        return this;
+    }
+
+    public int encodedLength() {
+        return ENCODED_LENGTH;
+    }
+
+    public static int blockLengthNullValue() {
+        return 65535;
+    }
+
+    public static int blockLengthMinValue() {
+        return 0;
+    }
+
+    public static int blockLengthMaxValue() {
+        return 65534;
+    }
+
+    public int blockLength() {
+        return (buffer.getShort(offset + 0, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF);
+    }
+
+
+    public static int numInGroupNullValue() {
+        return 65535;
+    }
+
+    public static int numInGroupMinValue() {
+        return 0;
+    }
+
+    public static int numInGroupMaxValue() {
+        return 65534;
+    }
+
+    public int numInGroup() {
+        return (buffer.getShort(offset + 2, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF);
+    }
+
+    public String toString() {
+        return appendTo(new StringBuilder(100)).toString();
+    }
+
+    public StringBuilder appendTo(final StringBuilder builder) {
+        builder.append('(');
+        //Token{signal=ENCODING, name='blockLength', description='Extra metadata bytes', id=-1, version=0, encodedLength=2, offset=0, componentTokenCount=1, encoding=Encoding{presence=REQUIRED, primitiveType=UINT16, byteOrder=LITTLE_ENDIAN, minValue=null, maxValue=null, nullValue=null, constValue=null, characterEncoding='UTF-8', epoch='null', timeUnit=null, semanticType='null'}}
+        builder.append("blockLength=");
+        builder.append(blockLength());
+        builder.append('|');
+        //Token{signal=ENCODING, name='numInGroup', description='Extra metadata bytes', id=-1, version=0, encodedLength=2, offset=2, componentTokenCount=1, encoding=Encoding{presence=REQUIRED, primitiveType=UINT16, byteOrder=LITTLE_ENDIAN, minValue=null, maxValue=null, nullValue=null, constValue=null, characterEncoding='UTF-8', epoch='null', timeUnit=null, semanticType='null'}}
+        builder.append("numInGroup=");
+        builder.append(numInGroup());
+        builder.append(')');
+
+        return builder;
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/GroupSizeEncodingEncoder.java

@@ -0,0 +1,88 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+import org.agrona.MutableDirectBuffer;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.GroupSizeEncodingEncoder"})
+@SuppressWarnings("all")
+public class GroupSizeEncodingEncoder {
+    public static final int ENCODED_LENGTH = 4;
+    private MutableDirectBuffer buffer;
+    private int offset;
+
+    public GroupSizeEncodingEncoder wrap(final MutableDirectBuffer buffer, final int offset) {
+        this.buffer = buffer;
+        this.offset = offset;
+
+        return this;
+    }
+
+    public int encodedLength() {
+        return ENCODED_LENGTH;
+    }
+
+    public static int blockLengthNullValue() {
+        return 65535;
+    }
+
+    public static int blockLengthMinValue() {
+        return 0;
+    }
+
+    public static int blockLengthMaxValue() {
+        return 65534;
+    }
+
+    public GroupSizeEncodingEncoder blockLength(final int value) {
+        buffer.putShort(offset + 0, (short) value, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+
+    public static int numInGroupNullValue() {
+        return 65535;
+    }
+
+    public static int numInGroupMinValue() {
+        return 0;
+    }
+
+    public static int numInGroupMaxValue() {
+        return 65534;
+    }
+
+    public GroupSizeEncodingEncoder numInGroup(final int value) {
+        buffer.putShort(offset + 2, (short) value, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public String toString() {
+        return appendTo(new StringBuilder(100)).toString();
+    }
+
+    public StringBuilder appendTo(final StringBuilder builder) {
+        GroupSizeEncodingDecoder writer = new GroupSizeEncodingDecoder();
+        writer.wrap(buffer, offset);
+
+        return writer.appendTo(builder);
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/InitFieldsPresentDecoder.java

@@ -0,0 +1,87 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+import org.agrona.DirectBuffer;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.InitFieldsPresentDecoder"})
+@SuppressWarnings("all")
+public class InitFieldsPresentDecoder {
+    public static final int ENCODED_LENGTH = 1;
+    private DirectBuffer buffer;
+    private int offset;
+
+    public InitFieldsPresentDecoder wrap(final DirectBuffer buffer, final int offset) {
+        this.buffer = buffer;
+        this.offset = offset;
+
+        return this;
+    }
+
+    public int encodedLength() {
+        return ENCODED_LENGTH;
+    }
+
+    public boolean softwareInfo() {
+        return 0 != (buffer.getByte(offset) & (1 << 0));
+    }
+
+    public boolean hardwareInfo() {
+        return 0 != (buffer.getByte(offset) & (1 << 1));
+    }
+
+    public boolean modelInfo() {
+        return 0 != (buffer.getByte(offset) & (1 << 2));
+    }
+
+    public String toString() {
+        return appendTo(new StringBuilder(100)).toString();
+    }
+
+    public StringBuilder appendTo(final StringBuilder builder) {
+        builder.append('{');
+        boolean atLeastOne = false;
+        if (softwareInfo()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("softwareInfo");
+            atLeastOne = true;
+        }
+        if (hardwareInfo()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("hardwareInfo");
+            atLeastOne = true;
+        }
+        if (modelInfo()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("modelInfo");
+            atLeastOne = true;
+        }
+        builder.append('}');
+
+        return builder;
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/InitFieldsPresentEncoder.java

@@ -0,0 +1,68 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+import org.agrona.MutableDirectBuffer;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.InitFieldsPresentEncoder"})
+@SuppressWarnings("all")
+public class InitFieldsPresentEncoder {
+    public static final int ENCODED_LENGTH = 1;
+    private MutableDirectBuffer buffer;
+    private int offset;
+
+    public InitFieldsPresentEncoder wrap(final MutableDirectBuffer buffer, final int offset) {
+        this.buffer = buffer;
+        this.offset = offset;
+
+        return this;
+    }
+
+    public int encodedLength() {
+        return ENCODED_LENGTH;
+    }
+
+    public InitFieldsPresentEncoder clear() {
+        buffer.putByte(offset, (byte) (short) 0);
+        return this;
+    }
+
+    public InitFieldsPresentEncoder softwareInfo(final boolean value) {
+        byte bits = buffer.getByte(offset);
+        bits = (byte) (value ? bits | (1 << 0) : bits & ~(1 << 0));
+        buffer.putByte(offset, bits);
+        return this;
+    }
+
+    public InitFieldsPresentEncoder hardwareInfo(final boolean value) {
+        byte bits = buffer.getByte(offset);
+        bits = (byte) (value ? bits | (1 << 1) : bits & ~(1 << 1));
+        buffer.putByte(offset, bits);
+        return this;
+    }
+
+    public InitFieldsPresentEncoder modelInfo(final boolean value) {
+        byte bits = buffer.getByte(offset);
+        bits = (byte) (value ? bits | (1 << 2) : bits & ~(1 << 2));
+        buffer.putByte(offset, bits);
+        return this;
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MemoryType.java

@@ -0,0 +1,60 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.MemoryType"})
+public enum MemoryType {
+    JvmCurrent((short) 0), JvmMax((short) 1), OffHeapCurrent((short) 2), OffHeapMax((short) 3), DeviceCurrent(
+                    (short) 4), DeviceMax((short) 5), NULL_VAL((short) 255);
+
+    private final short value;
+
+    MemoryType(final short value) {
+        this.value = value;
+    }
+
+    public short value() {
+        return value;
+    }
+
+    public static MemoryType get(final short value) {
+        switch (value) {
+            case 0:
+                return JvmCurrent;
+            case 1:
+                return JvmMax;
+            case 2:
+                return OffHeapCurrent;
+            case 3:
+                return OffHeapMax;
+            case 4:
+                return DeviceCurrent;
+            case 5:
+                return DeviceMax;
+        }
+
+        if ((short) 255 == value) {
+            return NULL_VAL;
+        }
+
+        throw new IllegalArgumentException("Unknown value: " + value);
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MessageHeaderDecoder.java

@@ -0,0 +1,110 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+import org.agrona.DirectBuffer;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.MessageHeaderDecoder"})
+@SuppressWarnings("all")
+public class MessageHeaderDecoder {
+    public static final int ENCODED_LENGTH = 8;
+    private DirectBuffer buffer;
+    private int offset;
+
+    public MessageHeaderDecoder wrap(final DirectBuffer buffer, final int offset) {
+        this.buffer = buffer;
+        this.offset = offset;
+
+        return this;
+    }
+
+    public int encodedLength() {
+        return ENCODED_LENGTH;
+    }
+
+    public static int blockLengthNullValue() {
+        return 65535;
+    }
+
+    public static int blockLengthMinValue() {
+        return 0;
+    }
+
+    public static int blockLengthMaxValue() {
+        return 65534;
+    }
+
+    public int blockLength() {
+        return (buffer.getShort(offset + 0, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF);
+    }
+
+
+    public static int templateIdNullValue() {
+        return 65535;
+    }
+
+    public static int templateIdMinValue() {
+        return 0;
+    }
+
+    public static int templateIdMaxValue() {
+        return 65534;
+    }
+
+    public int templateId() {
+        return (buffer.getShort(offset + 2, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF);
+    }
+
+
+    public static int schemaIdNullValue() {
+        return 65535;
+    }
+
+    public static int schemaIdMinValue() {
+        return 0;
+    }
+
+    public static int schemaIdMaxValue() {
+        return 65534;
+    }
+
+    public int schemaId() {
+        return (buffer.getShort(offset + 4, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF);
+    }
+
+
+    public static int versionNullValue() {
+        return 65535;
+    }
+
+    public static int versionMinValue() {
+        return 0;
+    }
+
+    public static int versionMaxValue() {
+        return 65534;
+    }
+
+    public int version() {
+        return (buffer.getShort(offset + 6, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF);
+    }
+
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MessageHeaderEncoder.java

@@ -0,0 +1,114 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+import org.agrona.MutableDirectBuffer;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.MessageHeaderEncoder"})
+@SuppressWarnings("all")
+public class MessageHeaderEncoder {
+    public static final int ENCODED_LENGTH = 8;
+    private MutableDirectBuffer buffer;
+    private int offset;
+
+    public MessageHeaderEncoder wrap(final MutableDirectBuffer buffer, final int offset) {
+        this.buffer = buffer;
+        this.offset = offset;
+
+        return this;
+    }
+
+    public int encodedLength() {
+        return ENCODED_LENGTH;
+    }
+
+    public static int blockLengthNullValue() {
+        return 65535;
+    }
+
+    public static int blockLengthMinValue() {
+        return 0;
+    }
+
+    public static int blockLengthMaxValue() {
+        return 65534;
+    }
+
+    public MessageHeaderEncoder blockLength(final int value) {
+        buffer.putShort(offset + 0, (short) value, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+
+    public static int templateIdNullValue() {
+        return 65535;
+    }
+
+    public static int templateIdMinValue() {
+        return 0;
+    }
+
+    public static int templateIdMaxValue() {
+        return 65534;
+    }
+
+    public MessageHeaderEncoder templateId(final int value) {
+        buffer.putShort(offset + 2, (short) value, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+
+    public static int schemaIdNullValue() {
+        return 65535;
+    }
+
+    public static int schemaIdMinValue() {
+        return 0;
+    }
+
+    public static int schemaIdMaxValue() {
+        return 65534;
+    }
+
+    public MessageHeaderEncoder schemaId(final int value) {
+        buffer.putShort(offset + 4, (short) value, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+
+    public static int versionNullValue() {
+        return 65535;
+    }
+
+    public static int versionMinValue() {
+        return 0;
+    }
+
+    public static int versionMaxValue() {
+        return 65534;
+    }
+
+    public MessageHeaderEncoder version(final int value) {
+        buffer.putShort(offset + 6, (short) value, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MetaAttribute.java

@@ -0,0 +1,26 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.MetaAttribute"})
+public enum MetaAttribute {
+    EPOCH, TIME_UNIT, SEMANTIC_TYPE
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StatSource.java

@@ -0,0 +1,53 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+@javax.annotation.Generated(value = {"StatSource"})
+public enum StatSource {
+    Parameters((short) 0), Updates((short) 1), Activations((short) 2), NULL_VAL((short) 255);
+
+    private final short value;
+
+    StatSource(final short value) {
+        this.value = value;
+    }
+
+    public short value() {
+        return value;
+    }
+
+    public static StatSource get(final short value) {
+        switch (value) {
+            case 0:
+                return Parameters;
+            case 1:
+                return Updates;
+            case 2:
+                return Activations;
+        }
+
+        if ((short) 255 == value) {
+            return NULL_VAL;
+        }
+
+        throw new IllegalArgumentException("Unknown value: " + value);
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StatType.java

@@ -0,0 +1,53 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+@javax.annotation.Generated(value = {"StatType"})
+public enum StatType {
+    Mean((short) 0), Stdev((short) 1), MeanMagnitude((short) 2), NULL_VAL((short) 255);
+
+    private final short value;
+
+    StatType(final short value) {
+        this.value = value;
+    }
+
+    public short value() {
+        return value;
+    }
+
+    public static StatType get(final short value) {
+        switch (value) {
+            case 0:
+                return Mean;
+            case 1:
+                return Stdev;
+            case 2:
+                return MeanMagnitude;
+        }
+
+        if ((short) 255 == value) {
+            return NULL_VAL;
+        }
+
+        throw new IllegalArgumentException("Unknown value: " + value);
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StatsType.java

@@ -0,0 +1,55 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.StatsType"})
+public enum StatsType {
+    Parameters((short) 0), Gradients((short) 1), Updates((short) 2), Activations((short) 3), NULL_VAL((short) 255);
+
+    private final short value;
+
+    StatsType(final short value) {
+        this.value = value;
+    }
+
+    public short value() {
+        return value;
+    }
+
+    public static StatsType get(final short value) {
+        switch (value) {
+            case 0:
+                return Parameters;
+            case 1:
+                return Gradients;
+            case 2:
+                return Updates;
+            case 3:
+                return Activations;
+        }
+
+        if ((short) 255 == value) {
+            return NULL_VAL;
+        }
+
+        throw new IllegalArgumentException("Unknown value: " + value);
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StorageMetaDataDecoder.java

@@ -0,0 +1,659 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+import org.agrona.DirectBuffer;
+import org.agrona.MutableDirectBuffer;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.StorageMetaDataDecoder"})
+@SuppressWarnings("all")
+public class StorageMetaDataDecoder {
+    public static final int BLOCK_LENGTH = 8;
+    public static final int TEMPLATE_ID = 3;
+    public static final int SCHEMA_ID = 1;
+    public static final int SCHEMA_VERSION = 0;
+
+    private final StorageMetaDataDecoder parentMessage = this;
+    private DirectBuffer buffer;
+    protected int offset;
+    protected int limit;
+    protected int actingBlockLength;
+    protected int actingVersion;
+
+    public int sbeBlockLength() {
+        return BLOCK_LENGTH;
+    }
+
+    public int sbeTemplateId() {
+        return TEMPLATE_ID;
+    }
+
+    public int sbeSchemaId() {
+        return SCHEMA_ID;
+    }
+
+    public int sbeSchemaVersion() {
+        return SCHEMA_VERSION;
+    }
+
+    public String sbeSemanticType() {
+        return "";
+    }
+
+    public int offset() {
+        return offset;
+    }
+
+    public StorageMetaDataDecoder wrap(final DirectBuffer buffer, final int offset, final int actingBlockLength,
+                    final int actingVersion) {
+        this.buffer = buffer;
+        this.offset = offset;
+        this.actingBlockLength = actingBlockLength;
+        this.actingVersion = actingVersion;
+        limit(offset + actingBlockLength);
+
+        return this;
+    }
+
+    public int encodedLength() {
+        return limit - offset;
+    }
+
+    public int limit() {
+        return limit;
+    }
+
+    public void limit(final int limit) {
+        this.limit = limit;
+    }
+
+    public static int timeStampId() {
+        return 1;
+    }
+
+    public static String timeStampMetaAttribute(final MetaAttribute metaAttribute) {
+        switch (metaAttribute) {
+            case EPOCH:
+                return "unix";
+            case TIME_UNIT:
+                return "nanosecond";
+            case SEMANTIC_TYPE:
+                return "";
+        }
+
+        return "";
+    }
+
+    public static long timeStampNullValue() {
+        return -9223372036854775808L;
+    }
+
+    public static long timeStampMinValue() {
+        return -9223372036854775807L;
+    }
+
+    public static long timeStampMaxValue() {
+        return 9223372036854775807L;
+    }
+
+    public long timeStamp() {
+        return buffer.getLong(offset + 0, java.nio.ByteOrder.LITTLE_ENDIAN);
+    }
+
+
+    private final ExtraMetaDataBytesDecoder extraMetaDataBytes = new ExtraMetaDataBytesDecoder();
+
+    public static long extraMetaDataBytesDecoderId() {
+        return 2;
+    }
+
+    public ExtraMetaDataBytesDecoder extraMetaDataBytes() {
+        extraMetaDataBytes.wrap(parentMessage, buffer);
+        return extraMetaDataBytes;
+    }
+
+    public static class ExtraMetaDataBytesDecoder
+                    implements Iterable<ExtraMetaDataBytesDecoder>, java.util.Iterator<ExtraMetaDataBytesDecoder> {
+        private static final int HEADER_SIZE = 4;
+        private final GroupSizeEncodingDecoder dimensions = new GroupSizeEncodingDecoder();
+        private StorageMetaDataDecoder parentMessage;
+        private DirectBuffer buffer;
+        private int blockLength;
+        private int actingVersion;
+        private int count;
+        private int index;
+        private int offset;
+
+        public void wrap(final StorageMetaDataDecoder parentMessage, final DirectBuffer buffer) {
+            this.parentMessage = parentMessage;
+            this.buffer = buffer;
+            dimensions.wrap(buffer, parentMessage.limit());
+            blockLength = dimensions.blockLength();
+            count = dimensions.numInGroup();
+            index = -1;
+            parentMessage.limit(parentMessage.limit() + HEADER_SIZE);
+        }
+
+        public static int sbeHeaderSize() {
+            return HEADER_SIZE;
+        }
+
+        public static int sbeBlockLength() {
+            return 1;
+        }
+
+        public int actingBlockLength() {
+            return blockLength;
+        }
+
+        public int count() {
+            return count;
+        }
+
+        public java.util.Iterator<ExtraMetaDataBytesDecoder> iterator() {
+            return this;
+        }
+
+        public void remove() {
+            throw new UnsupportedOperationException();
+        }
+
+        public boolean hasNext() {
+            return (index + 1) < count;
+        }
+
+        public ExtraMetaDataBytesDecoder next() {
+            if (index + 1 >= count) {
+                throw new java.util.NoSuchElementException();
+            }
+
+            offset = parentMessage.limit();
+            parentMessage.limit(offset + blockLength);
+            ++index;
+
+            return this;
+        }
+
+        public static int bytesId() {
+            return 3;
+        }
+
+        public static String bytesMetaAttribute(final MetaAttribute metaAttribute) {
+            switch (metaAttribute) {
+                case EPOCH:
+                    return "unix";
+                case TIME_UNIT:
+                    return "nanosecond";
+                case SEMANTIC_TYPE:
+                    return "";
+            }
+
+            return "";
+        }
+
+        public static byte bytesNullValue() {
+            return (byte) -128;
+        }
+
+        public static byte bytesMinValue() {
+            return (byte) -127;
+        }
+
+        public static byte bytesMaxValue() {
+            return (byte) 127;
+        }
+
+        public byte bytes() {
+            return buffer.getByte(offset + 0);
+        }
+
+
+        public String toString() {
+            return appendTo(new StringBuilder(100)).toString();
+        }
+
+        public StringBuilder appendTo(final StringBuilder builder) {
+            builder.append('(');
+            //Token{signal=BEGIN_FIELD, name='bytes', description='null', id=3, version=0, encodedLength=0, offset=0, componentTokenCount=3, encoding=Encoding{presence=REQUIRED, primitiveType=null, byteOrder=LITTLE_ENDIAN, minValue=null, maxValue=null, nullValue=null, constValue=null, characterEncoding='null', epoch='unix', timeUnit=nanosecond, semanticType='null'}}
+            //Token{signal=ENCODING, name='int8', description='null', id=-1, version=0, encodedLength=1, offset=0, componentTokenCount=1, encoding=Encoding{presence=REQUIRED, primitiveType=INT8, byteOrder=LITTLE_ENDIAN, minValue=null, maxValue=null, nullValue=null, constValue=null, characterEncoding='null', epoch='unix', timeUnit=nanosecond, semanticType='null'}}
+            builder.append("bytes=");
+            builder.append(bytes());
+            builder.append(')');
+            return builder;
+        }
+    }
+
+    public static int sessionIDId() {
+        return 4;
+    }
+
+    public static String sessionIDCharacterEncoding() {
+        return "UTF-8";
+    }
+
+    public static String sessionIDMetaAttribute(final MetaAttribute metaAttribute) {
+        switch (metaAttribute) {
+            case EPOCH:
+                return "unix";
+            case TIME_UNIT:
+                return "nanosecond";
+            case SEMANTIC_TYPE:
+                return "";
+        }
+
+        return "";
+    }
+
+    public static int sessionIDHeaderLength() {
+        return 4;
+    }
+
+    public int sessionIDLength() {
+        final int limit = parentMessage.limit();
+        return (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+    }
+
+    public int getSessionID(final MutableDirectBuffer dst, final int dstOffset, final int length) {
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        final int dataLength = (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+        final int bytesCopied = Math.min(length, dataLength);
+        parentMessage.limit(limit + headerLength + dataLength);
+        buffer.getBytes(limit + headerLength, dst, dstOffset, bytesCopied);
+
+        return bytesCopied;
+    }
+
+    public int getSessionID(final byte[] dst, final int dstOffset, final int length) {
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        final int dataLength = (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+        final int bytesCopied = Math.min(length, dataLength);
+        parentMessage.limit(limit + headerLength + dataLength);
+        buffer.getBytes(limit + headerLength, dst, dstOffset, bytesCopied);
+
+        return bytesCopied;
+    }
+
+    public String sessionID() {
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        final int dataLength = (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+        parentMessage.limit(limit + headerLength + dataLength);
+        final byte[] tmp = new byte[dataLength];
+        buffer.getBytes(limit + headerLength, tmp, 0, dataLength);
+
+        final String value;
+        try {
+            value = new String(tmp, "UTF-8");
+        } catch (final java.io.UnsupportedEncodingException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        return value;
+    }
+
+    public static int typeIDId() {
+        return 5;
+    }
+
+    public static String typeIDCharacterEncoding() {
+        return "UTF-8";
+    }
+
+    public static String typeIDMetaAttribute(final MetaAttribute metaAttribute) {
+        switch (metaAttribute) {
+            case EPOCH:
+                return "unix";
+            case TIME_UNIT:
+                return "nanosecond";
+            case SEMANTIC_TYPE:
+                return "";
+        }
+
+        return "";
+    }
+
+    public static int typeIDHeaderLength() {
+        return 4;
+    }
+
+    public int typeIDLength() {
+        final int limit = parentMessage.limit();
+        return (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+    }
+
+    public int getTypeID(final MutableDirectBuffer dst, final int dstOffset, final int length) {
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        final int dataLength = (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+        final int bytesCopied = Math.min(length, dataLength);
+        parentMessage.limit(limit + headerLength + dataLength);
+        buffer.getBytes(limit + headerLength, dst, dstOffset, bytesCopied);
+
+        return bytesCopied;
+    }
+
+    public int getTypeID(final byte[] dst, final int dstOffset, final int length) {
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        final int dataLength = (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+        final int bytesCopied = Math.min(length, dataLength);
+        parentMessage.limit(limit + headerLength + dataLength);
+        buffer.getBytes(limit + headerLength, dst, dstOffset, bytesCopied);
+
+        return bytesCopied;
+    }
+
+    public String typeID() {
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        final int dataLength = (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+        parentMessage.limit(limit + headerLength + dataLength);
+        final byte[] tmp = new byte[dataLength];
+        buffer.getBytes(limit + headerLength, tmp, 0, dataLength);
+
+        final String value;
+        try {
+            value = new String(tmp, "UTF-8");
+        } catch (final java.io.UnsupportedEncodingException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        return value;
+    }
+
+    public static int workerIDId() {
+        return 6;
+    }
+
+    public static String workerIDCharacterEncoding() {
+        return "UTF-8";
+    }
+
+    public static String workerIDMetaAttribute(final MetaAttribute metaAttribute) {
+        switch (metaAttribute) {
+            case EPOCH:
+                return "unix";
+            case TIME_UNIT:
+                return "nanosecond";
+            case SEMANTIC_TYPE:
+                return "";
+        }
+
+        return "";
+    }
+
+    public static int workerIDHeaderLength() {
+        return 4;
+    }
+
+    public int workerIDLength() {
+        final int limit = parentMessage.limit();
+        return (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+    }
+
+    public int getWorkerID(final MutableDirectBuffer dst, final int dstOffset, final int length) {
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        final int dataLength = (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+        final int bytesCopied = Math.min(length, dataLength);
+        parentMessage.limit(limit + headerLength + dataLength);
+        buffer.getBytes(limit + headerLength, dst, dstOffset, bytesCopied);
+
+        return bytesCopied;
+    }
+
+    public int getWorkerID(final byte[] dst, final int dstOffset, final int length) {
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        final int dataLength = (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+        final int bytesCopied = Math.min(length, dataLength);
+        parentMessage.limit(limit + headerLength + dataLength);
+        buffer.getBytes(limit + headerLength, dst, dstOffset, bytesCopied);
+
+        return bytesCopied;
+    }
+
+    public String workerID() {
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        final int dataLength = (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+        parentMessage.limit(limit + headerLength + dataLength);
+        final byte[] tmp = new byte[dataLength];
+        buffer.getBytes(limit + headerLength, tmp, 0, dataLength);
+
+        final String value;
+        try {
+            value = new String(tmp, "UTF-8");
+        } catch (final java.io.UnsupportedEncodingException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        return value;
+    }
+
+    public static int initTypeClassId() {
+        return 7;
+    }
+
+    public static String initTypeClassCharacterEncoding() {
+        return "UTF-8";
+    }
+
+    public static String initTypeClassMetaAttribute(final MetaAttribute metaAttribute) {
+        switch (metaAttribute) {
+            case EPOCH:
+                return "unix";
+            case TIME_UNIT:
+                return "nanosecond";
+            case SEMANTIC_TYPE:
+                return "";
+        }
+
+        return "";
+    }
+
+    public static int initTypeClassHeaderLength() {
+        return 4;
+    }
+
+    public int initTypeClassLength() {
+        final int limit = parentMessage.limit();
+        return (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+    }
+
+    public int getInitTypeClass(final MutableDirectBuffer dst, final int dstOffset, final int length) {
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        final int dataLength = (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+        final int bytesCopied = Math.min(length, dataLength);
+        parentMessage.limit(limit + headerLength + dataLength);
+        buffer.getBytes(limit + headerLength, dst, dstOffset, bytesCopied);
+
+        return bytesCopied;
+    }
+
+    public int getInitTypeClass(final byte[] dst, final int dstOffset, final int length) {
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        final int dataLength = (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+        final int bytesCopied = Math.min(length, dataLength);
+        parentMessage.limit(limit + headerLength + dataLength);
+        buffer.getBytes(limit + headerLength, dst, dstOffset, bytesCopied);
+
+        return bytesCopied;
+    }
+
+    public String initTypeClass() {
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        final int dataLength = (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+        parentMessage.limit(limit + headerLength + dataLength);
+        final byte[] tmp = new byte[dataLength];
+        buffer.getBytes(limit + headerLength, tmp, 0, dataLength);
+
+        final String value;
+        try {
+            value = new String(tmp, "UTF-8");
+        } catch (final java.io.UnsupportedEncodingException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        return value;
+    }
+
+    public static int updateTypeClassId() {
+        return 8;
+    }
+
+    public static String updateTypeClassCharacterEncoding() {
+        return "UTF-8";
+    }
+
+    public static String updateTypeClassMetaAttribute(final MetaAttribute metaAttribute) {
+        switch (metaAttribute) {
+            case EPOCH:
+                return "unix";
+            case TIME_UNIT:
+                return "nanosecond";
+            case SEMANTIC_TYPE:
+                return "";
+        }
+
+        return "";
+    }
+
+    public static int updateTypeClassHeaderLength() {
+        return 4;
+    }
+
+    public int updateTypeClassLength() {
+        final int limit = parentMessage.limit();
+        return (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+    }
+
+    public int getUpdateTypeClass(final MutableDirectBuffer dst, final int dstOffset, final int length) {
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        final int dataLength = (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+        final int bytesCopied = Math.min(length, dataLength);
+        parentMessage.limit(limit + headerLength + dataLength);
+        buffer.getBytes(limit + headerLength, dst, dstOffset, bytesCopied);
+
+        return bytesCopied;
+    }
+
+    public int getUpdateTypeClass(final byte[] dst, final int dstOffset, final int length) {
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        final int dataLength = (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+        final int bytesCopied = Math.min(length, dataLength);
+        parentMessage.limit(limit + headerLength + dataLength);
+        buffer.getBytes(limit + headerLength, dst, dstOffset, bytesCopied);
+
+        return bytesCopied;
+    }
+
+    public String updateTypeClass() {
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        final int dataLength = (int) (buffer.getInt(limit, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+        parentMessage.limit(limit + headerLength + dataLength);
+        final byte[] tmp = new byte[dataLength];
+        buffer.getBytes(limit + headerLength, tmp, 0, dataLength);
+
+        final String value;
+        try {
+            value = new String(tmp, "UTF-8");
+        } catch (final java.io.UnsupportedEncodingException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        return value;
+    }
+
+    public String toString() {
+        return appendTo(new StringBuilder(100)).toString();
+    }
+
+    public StringBuilder appendTo(final StringBuilder builder) {
+        final int originalLimit = limit();
+        limit(offset + actingBlockLength);
+        builder.append("[StorageMetaData](sbeTemplateId=");
+        builder.append(TEMPLATE_ID);
+        builder.append("|sbeSchemaId=");
+        builder.append(SCHEMA_ID);
+        builder.append("|sbeSchemaVersion=");
+        if (actingVersion != SCHEMA_VERSION) {
+            builder.append(actingVersion);
+            builder.append('/');
+        }
+        builder.append(SCHEMA_VERSION);
+        builder.append("|sbeBlockLength=");
+        if (actingBlockLength != BLOCK_LENGTH) {
+            builder.append(actingBlockLength);
+            builder.append('/');
+        }
+        builder.append(BLOCK_LENGTH);
+        builder.append("):");
+        //Token{signal=BEGIN_FIELD, name='timeStamp', description='null', id=1, version=0, encodedLength=0, offset=0, componentTokenCount=3, encoding=Encoding{presence=REQUIRED, primitiveType=null, byteOrder=LITTLE_ENDIAN, minValue=null, maxValue=null, nullValue=null, constValue=null, characterEncoding='null', epoch='unix', timeUnit=nanosecond, semanticType='null'}}
+        //Token{signal=ENCODING, name='int64', description='null', id=-1, version=0, encodedLength=8, offset=0, componentTokenCount=1, encoding=Encoding{presence=REQUIRED, primitiveType=INT64, byteOrder=LITTLE_ENDIAN, minValue=null, maxValue=null, nullValue=null, constValue=null, characterEncoding='null', epoch='unix', timeUnit=nanosecond, semanticType='null'}}
+        builder.append("timeStamp=");
+        builder.append(timeStamp());
+        builder.append('|');
+        //Token{signal=BEGIN_GROUP, name='extraMetaDataBytes', description='Extra metadata bytes', id=2, version=0, encodedLength=1, offset=8, componentTokenCount=9, encoding=Encoding{presence=REQUIRED, primitiveType=null, byteOrder=LITTLE_ENDIAN, minValue=null, maxValue=null, nullValue=null, constValue=null, characterEncoding='null', epoch='null', timeUnit=null, semanticType='null'}}
+        builder.append("extraMetaDataBytes=[");
+        ExtraMetaDataBytesDecoder extraMetaDataBytes = extraMetaDataBytes();
+        if (extraMetaDataBytes.count() > 0) {
+            while (extraMetaDataBytes.hasNext()) {
+                extraMetaDataBytes.next().appendTo(builder);
+                builder.append(',');
+            }
+            builder.setLength(builder.length() - 1);
+        }
+        builder.append(']');
+        builder.append('|');
+        //Token{signal=BEGIN_VAR_DATA, name='sessionID', description='null', id=4, version=0, encodedLength=0, offset=-1, componentTokenCount=6, encoding=Encoding{presence=REQUIRED, primitiveType=null, byteOrder=LITTLE_ENDIAN, minValue=null, maxValue=null, nullValue=null, constValue=null, characterEncoding='null', epoch='unix', timeUnit=nanosecond, semanticType='null'}}
+        builder.append("sessionID=");
+        builder.append(sessionID());
+        builder.append('|');
+        //Token{signal=BEGIN_VAR_DATA, name='typeID', description='null', id=5, version=0, encodedLength=0, offset=-1, componentTokenCount=6, encoding=Encoding{presence=REQUIRED, primitiveType=null, byteOrder=LITTLE_ENDIAN, minValue=null, maxValue=null, nullValue=null, constValue=null, characterEncoding='null', epoch='unix', timeUnit=nanosecond, semanticType='null'}}
+        builder.append("typeID=");
+        builder.append(typeID());
+        builder.append('|');
+        //Token{signal=BEGIN_VAR_DATA, name='workerID', description='null', id=6, version=0, encodedLength=0, offset=-1, componentTokenCount=6, encoding=Encoding{presence=REQUIRED, primitiveType=null, byteOrder=LITTLE_ENDIAN, minValue=null, maxValue=null, nullValue=null, constValue=null, characterEncoding='null', epoch='unix', timeUnit=nanosecond, semanticType='null'}}
+        builder.append("workerID=");
+        builder.append(workerID());
+        builder.append('|');
+        //Token{signal=BEGIN_VAR_DATA, name='initTypeClass', description='null', id=7, version=0, encodedLength=0, offset=-1, componentTokenCount=6, encoding=Encoding{presence=REQUIRED, primitiveType=null, byteOrder=LITTLE_ENDIAN, minValue=null, maxValue=null, nullValue=null, constValue=null, characterEncoding='null', epoch='unix', timeUnit=nanosecond, semanticType='null'}}
+        builder.append("initTypeClass=");
+        builder.append(initTypeClass());
+        builder.append('|');
+        //Token{signal=BEGIN_VAR_DATA, name='updateTypeClass', description='null', id=8, version=0, encodedLength=0, offset=-1, componentTokenCount=6, encoding=Encoding{presence=REQUIRED, primitiveType=null, byteOrder=LITTLE_ENDIAN, minValue=null, maxValue=null, nullValue=null, constValue=null, characterEncoding='null', epoch='unix', timeUnit=nanosecond, semanticType='null'}}
+        builder.append("updateTypeClass=");
+        builder.append(updateTypeClass());
+
+        limit(originalLimit);
+
+        return builder;
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StorageMetaDataEncoder.java

@@ -0,0 +1,567 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+import org.agrona.DirectBuffer;
+import org.agrona.MutableDirectBuffer;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.StorageMetaDataEncoder"})
+@SuppressWarnings("all")
+public class StorageMetaDataEncoder {
+    public static final int BLOCK_LENGTH = 8;
+    public static final int TEMPLATE_ID = 3;
+    public static final int SCHEMA_ID = 1;
+    public static final int SCHEMA_VERSION = 0;
+
+    private final StorageMetaDataEncoder parentMessage = this;
+    private MutableDirectBuffer buffer;
+    protected int offset;
+    protected int limit;
+    protected int actingBlockLength;
+    protected int actingVersion;
+
+    public int sbeBlockLength() {
+        return BLOCK_LENGTH;
+    }
+
+    public int sbeTemplateId() {
+        return TEMPLATE_ID;
+    }
+
+    public int sbeSchemaId() {
+        return SCHEMA_ID;
+    }
+
+    public int sbeSchemaVersion() {
+        return SCHEMA_VERSION;
+    }
+
+    public String sbeSemanticType() {
+        return "";
+    }
+
+    public int offset() {
+        return offset;
+    }
+
+    public StorageMetaDataEncoder wrap(final MutableDirectBuffer buffer, final int offset) {
+        this.buffer = buffer;
+        this.offset = offset;
+        limit(offset + BLOCK_LENGTH);
+
+        return this;
+    }
+
+    public int encodedLength() {
+        return limit - offset;
+    }
+
+    public int limit() {
+        return limit;
+    }
+
+    public void limit(final int limit) {
+        this.limit = limit;
+    }
+
+    public static long timeStampNullValue() {
+        return -9223372036854775808L;
+    }
+
+    public static long timeStampMinValue() {
+        return -9223372036854775807L;
+    }
+
+    public static long timeStampMaxValue() {
+        return 9223372036854775807L;
+    }
+
+    public StorageMetaDataEncoder timeStamp(final long value) {
+        buffer.putLong(offset + 0, value, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+
+    private final ExtraMetaDataBytesEncoder extraMetaDataBytes = new ExtraMetaDataBytesEncoder();
+
+    public static long extraMetaDataBytesId() {
+        return 2;
+    }
+
+    public ExtraMetaDataBytesEncoder extraMetaDataBytesCount(final int count) {
+        extraMetaDataBytes.wrap(parentMessage, buffer, count);
+        return extraMetaDataBytes;
+    }
+
+    public static class ExtraMetaDataBytesEncoder {
+        private static final int HEADER_SIZE = 4;
+        private final GroupSizeEncodingEncoder dimensions = new GroupSizeEncodingEncoder();
+        private StorageMetaDataEncoder parentMessage;
+        private MutableDirectBuffer buffer;
+        private int blockLength;
+        private int actingVersion;
+        private int count;
+        private int index;
+        private int offset;
+
+        public void wrap(final StorageMetaDataEncoder parentMessage, final MutableDirectBuffer buffer,
+                        final int count) {
+            if (count < 0 || count > 65534) {
+                throw new IllegalArgumentException("count outside allowed range: count=" + count);
+            }
+
+            this.parentMessage = parentMessage;
+            this.buffer = buffer;
+            actingVersion = SCHEMA_VERSION;
+            dimensions.wrap(buffer, parentMessage.limit());
+            dimensions.blockLength((int) 1);
+            dimensions.numInGroup((int) count);
+            index = -1;
+            this.count = count;
+            blockLength = 1;
+            parentMessage.limit(parentMessage.limit() + HEADER_SIZE);
+        }
+
+        public static int sbeHeaderSize() {
+            return HEADER_SIZE;
+        }
+
+        public static int sbeBlockLength() {
+            return 1;
+        }
+
+        public ExtraMetaDataBytesEncoder next() {
+            if (index + 1 >= count) {
+                throw new java.util.NoSuchElementException();
+            }
+
+            offset = parentMessage.limit();
+            parentMessage.limit(offset + blockLength);
+            ++index;
+
+            return this;
+        }
+
+        public static byte bytesNullValue() {
+            return (byte) -128;
+        }
+
+        public static byte bytesMinValue() {
+            return (byte) -127;
+        }
+
+        public static byte bytesMaxValue() {
+            return (byte) 127;
+        }
+
+        public ExtraMetaDataBytesEncoder bytes(final byte value) {
+            buffer.putByte(offset + 0, value);
+            return this;
+        }
+
+    }
+
+    public static int sessionIDId() {
+        return 4;
+    }
+
+    public static String sessionIDCharacterEncoding() {
+        return "UTF-8";
+    }
+
+    public static String sessionIDMetaAttribute(final MetaAttribute metaAttribute) {
+        switch (metaAttribute) {
+            case EPOCH:
+                return "unix";
+            case TIME_UNIT:
+                return "nanosecond";
+            case SEMANTIC_TYPE:
+                return "";
+        }
+
+        return "";
+    }
+
+    public static int sessionIDHeaderLength() {
+        return 4;
+    }
+
+    public StorageMetaDataEncoder putSessionID(final DirectBuffer src, final int srcOffset, final int length) {
+        if (length > 1073741824) {
+            throw new IllegalArgumentException("length > max value for type: " + length);
+        }
+
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        parentMessage.limit(limit + headerLength + length);
+        buffer.putInt(limit, (int) length, java.nio.ByteOrder.LITTLE_ENDIAN);
+        buffer.putBytes(limit + headerLength, src, srcOffset, length);
+
+        return this;
+    }
+
+    public StorageMetaDataEncoder putSessionID(final byte[] src, final int srcOffset, final int length) {
+        if (length > 1073741824) {
+            throw new IllegalArgumentException("length > max value for type: " + length);
+        }
+
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        parentMessage.limit(limit + headerLength + length);
+        buffer.putInt(limit, (int) length, java.nio.ByteOrder.LITTLE_ENDIAN);
+        buffer.putBytes(limit + headerLength, src, srcOffset, length);
+
+        return this;
+    }
+
+    public StorageMetaDataEncoder sessionID(final String value) {
+        final byte[] bytes;
+        try {
+            bytes = value.getBytes("UTF-8");
+        } catch (final java.io.UnsupportedEncodingException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        final int length = bytes.length;
+        if (length > 1073741824) {
+            throw new IllegalArgumentException("length > max value for type: " + length);
+        }
+
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        parentMessage.limit(limit + headerLength + length);
+        buffer.putInt(limit, (int) length, java.nio.ByteOrder.LITTLE_ENDIAN);
+        buffer.putBytes(limit + headerLength, bytes, 0, length);
+
+        return this;
+    }
+
+    public static int typeIDId() {
+        return 5;
+    }
+
+    public static String typeIDCharacterEncoding() {
+        return "UTF-8";
+    }
+
+    public static String typeIDMetaAttribute(final MetaAttribute metaAttribute) {
+        switch (metaAttribute) {
+            case EPOCH:
+                return "unix";
+            case TIME_UNIT:
+                return "nanosecond";
+            case SEMANTIC_TYPE:
+                return "";
+        }
+
+        return "";
+    }
+
+    public static int typeIDHeaderLength() {
+        return 4;
+    }
+
+    public StorageMetaDataEncoder putTypeID(final DirectBuffer src, final int srcOffset, final int length) {
+        if (length > 1073741824) {
+            throw new IllegalArgumentException("length > max value for type: " + length);
+        }
+
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        parentMessage.limit(limit + headerLength + length);
+        buffer.putInt(limit, (int) length, java.nio.ByteOrder.LITTLE_ENDIAN);
+        buffer.putBytes(limit + headerLength, src, srcOffset, length);
+
+        return this;
+    }
+
+    public StorageMetaDataEncoder putTypeID(final byte[] src, final int srcOffset, final int length) {
+        if (length > 1073741824) {
+            throw new IllegalArgumentException("length > max value for type: " + length);
+        }
+
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        parentMessage.limit(limit + headerLength + length);
+        buffer.putInt(limit, (int) length, java.nio.ByteOrder.LITTLE_ENDIAN);
+        buffer.putBytes(limit + headerLength, src, srcOffset, length);
+
+        return this;
+    }
+
+    public StorageMetaDataEncoder typeID(final String value) {
+        final byte[] bytes;
+        try {
+            bytes = value.getBytes("UTF-8");
+        } catch (final java.io.UnsupportedEncodingException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        final int length = bytes.length;
+        if (length > 1073741824) {
+            throw new IllegalArgumentException("length > max value for type: " + length);
+        }
+
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        parentMessage.limit(limit + headerLength + length);
+        buffer.putInt(limit, (int) length, java.nio.ByteOrder.LITTLE_ENDIAN);
+        buffer.putBytes(limit + headerLength, bytes, 0, length);
+
+        return this;
+    }
+
+    public static int workerIDId() {
+        return 6;
+    }
+
+    public static String workerIDCharacterEncoding() {
+        return "UTF-8";
+    }
+
+    public static String workerIDMetaAttribute(final MetaAttribute metaAttribute) {
+        switch (metaAttribute) {
+            case EPOCH:
+                return "unix";
+            case TIME_UNIT:
+                return "nanosecond";
+            case SEMANTIC_TYPE:
+                return "";
+        }
+
+        return "";
+    }
+
+    public static int workerIDHeaderLength() {
+        return 4;
+    }
+
+    public StorageMetaDataEncoder putWorkerID(final DirectBuffer src, final int srcOffset, final int length) {
+        if (length > 1073741824) {
+            throw new IllegalArgumentException("length > max value for type: " + length);
+        }
+
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        parentMessage.limit(limit + headerLength + length);
+        buffer.putInt(limit, (int) length, java.nio.ByteOrder.LITTLE_ENDIAN);
+        buffer.putBytes(limit + headerLength, src, srcOffset, length);
+
+        return this;
+    }
+
+    public StorageMetaDataEncoder putWorkerID(final byte[] src, final int srcOffset, final int length) {
+        if (length > 1073741824) {
+            throw new IllegalArgumentException("length > max value for type: " + length);
+        }
+
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        parentMessage.limit(limit + headerLength + length);
+        buffer.putInt(limit, (int) length, java.nio.ByteOrder.LITTLE_ENDIAN);
+        buffer.putBytes(limit + headerLength, src, srcOffset, length);
+
+        return this;
+    }
+
+    public StorageMetaDataEncoder workerID(final String value) {
+        final byte[] bytes;
+        try {
+            bytes = value.getBytes("UTF-8");
+        } catch (final java.io.UnsupportedEncodingException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        final int length = bytes.length;
+        if (length > 1073741824) {
+            throw new IllegalArgumentException("length > max value for type: " + length);
+        }
+
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        parentMessage.limit(limit + headerLength + length);
+        buffer.putInt(limit, (int) length, java.nio.ByteOrder.LITTLE_ENDIAN);
+        buffer.putBytes(limit + headerLength, bytes, 0, length);
+
+        return this;
+    }
+
+    public static int initTypeClassId() {
+        return 7;
+    }
+
+    public static String initTypeClassCharacterEncoding() {
+        return "UTF-8";
+    }
+
+    public static String initTypeClassMetaAttribute(final MetaAttribute metaAttribute) {
+        switch (metaAttribute) {
+            case EPOCH:
+                return "unix";
+            case TIME_UNIT:
+                return "nanosecond";
+            case SEMANTIC_TYPE:
+                return "";
+        }
+
+        return "";
+    }
+
+    public static int initTypeClassHeaderLength() {
+        return 4;
+    }
+
+    public StorageMetaDataEncoder putInitTypeClass(final DirectBuffer src, final int srcOffset, final int length) {
+        if (length > 1073741824) {
+            throw new IllegalArgumentException("length > max value for type: " + length);
+        }
+
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        parentMessage.limit(limit + headerLength + length);
+        buffer.putInt(limit, (int) length, java.nio.ByteOrder.LITTLE_ENDIAN);
+        buffer.putBytes(limit + headerLength, src, srcOffset, length);
+
+        return this;
+    }
+
+    public StorageMetaDataEncoder putInitTypeClass(final byte[] src, final int srcOffset, final int length) {
+        if (length > 1073741824) {
+            throw new IllegalArgumentException("length > max value for type: " + length);
+        }
+
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        parentMessage.limit(limit + headerLength + length);
+        buffer.putInt(limit, (int) length, java.nio.ByteOrder.LITTLE_ENDIAN);
+        buffer.putBytes(limit + headerLength, src, srcOffset, length);
+
+        return this;
+    }
+
+    public StorageMetaDataEncoder initTypeClass(final String value) {
+        final byte[] bytes;
+        try {
+            bytes = value.getBytes("UTF-8");
+        } catch (final java.io.UnsupportedEncodingException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        final int length = bytes.length;
+        if (length > 1073741824) {
+            throw new IllegalArgumentException("length > max value for type: " + length);
+        }
+
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        parentMessage.limit(limit + headerLength + length);
+        buffer.putInt(limit, (int) length, java.nio.ByteOrder.LITTLE_ENDIAN);
+        buffer.putBytes(limit + headerLength, bytes, 0, length);
+
+        return this;
+    }
+
+    public static int updateTypeClassId() {
+        return 8;
+    }
+
+    public static String updateTypeClassCharacterEncoding() {
+        return "UTF-8";
+    }
+
+    public static String updateTypeClassMetaAttribute(final MetaAttribute metaAttribute) {
+        switch (metaAttribute) {
+            case EPOCH:
+                return "unix";
+            case TIME_UNIT:
+                return "nanosecond";
+            case SEMANTIC_TYPE:
+                return "";
+        }
+
+        return "";
+    }
+
+    public static int updateTypeClassHeaderLength() {
+        return 4;
+    }
+
+    public StorageMetaDataEncoder putUpdateTypeClass(final DirectBuffer src, final int srcOffset, final int length) {
+        if (length > 1073741824) {
+            throw new IllegalArgumentException("length > max value for type: " + length);
+        }
+
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        parentMessage.limit(limit + headerLength + length);
+        buffer.putInt(limit, (int) length, java.nio.ByteOrder.LITTLE_ENDIAN);
+        buffer.putBytes(limit + headerLength, src, srcOffset, length);
+
+        return this;
+    }
+
+    public StorageMetaDataEncoder putUpdateTypeClass(final byte[] src, final int srcOffset, final int length) {
+        if (length > 1073741824) {
+            throw new IllegalArgumentException("length > max value for type: " + length);
+        }
+
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        parentMessage.limit(limit + headerLength + length);
+        buffer.putInt(limit, (int) length, java.nio.ByteOrder.LITTLE_ENDIAN);
+        buffer.putBytes(limit + headerLength, src, srcOffset, length);
+
+        return this;
+    }
+
+    public StorageMetaDataEncoder updateTypeClass(final String value) {
+        final byte[] bytes;
+        try {
+            bytes = value.getBytes("UTF-8");
+        } catch (final java.io.UnsupportedEncodingException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        final int length = bytes.length;
+        if (length > 1073741824) {
+            throw new IllegalArgumentException("length > max value for type: " + length);
+        }
+
+        final int headerLength = 4;
+        final int limit = parentMessage.limit();
+        parentMessage.limit(limit + headerLength + length);
+        buffer.putInt(limit, (int) length, java.nio.ByteOrder.LITTLE_ENDIAN);
+        buffer.putBytes(limit + headerLength, bytes, 0, length);
+
+        return this;
+    }
+
+    public String toString() {
+        return appendTo(new StringBuilder(100)).toString();
+    }
+
+    public StringBuilder appendTo(final StringBuilder builder) {
+        StorageMetaDataDecoder writer = new StorageMetaDataDecoder();
+        writer.wrap(buffer, offset, BLOCK_LENGTH, SCHEMA_VERSION);
+
+        return writer.appendTo(builder);
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/SummaryType.java

@@ -0,0 +1,53 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.SummaryType"})
+public enum SummaryType {
+    Mean((short) 0), Stdev((short) 1), MeanMagnitude((short) 2), NULL_VAL((short) 255);
+
+    private final short value;
+
+    SummaryType(final short value) {
+        this.value = value;
+    }
+
+    public short value() {
+        return value;
+    }
+
+    public static SummaryType get(final short value) {
+        switch (value) {
+            case 0:
+                return Mean;
+            case 1:
+                return Stdev;
+            case 2:
+                return MeanMagnitude;
+        }
+
+        if ((short) 255 == value) {
+            return NULL_VAL;
+        }
+
+        throw new IllegalArgumentException("Unknown value: " + value);
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateFieldsPresentDecoder.java

@@ -0,0 +1,296 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+import org.agrona.DirectBuffer;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.UpdateFieldsPresentDecoder"})
+@SuppressWarnings("all")
+public class UpdateFieldsPresentDecoder {
+    public static final int ENCODED_LENGTH = 4;
+    private DirectBuffer buffer;
+    private int offset;
+
+    public UpdateFieldsPresentDecoder wrap(final DirectBuffer buffer, final int offset) {
+        this.buffer = buffer;
+        this.offset = offset;
+
+        return this;
+    }
+
+    public int encodedLength() {
+        return ENCODED_LENGTH;
+    }
+
+    public boolean score() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 0));
+    }
+
+    public boolean memoryUse() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 1));
+    }
+
+    public boolean performance() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 2));
+    }
+
+    public boolean garbageCollection() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 3));
+    }
+
+    public boolean histogramParameters() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 4));
+    }
+
+    public boolean histogramGradients() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 5));
+    }
+
+    public boolean histogramUpdates() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 6));
+    }
+
+    public boolean histogramActivations() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 7));
+    }
+
+    public boolean meanParameters() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 8));
+    }
+
+    public boolean meanGradients() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 9));
+    }
+
+    public boolean meanUpdates() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 10));
+    }
+
+    public boolean meanActivations() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 11));
+    }
+
+    public boolean stdevParameters() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 12));
+    }
+
+    public boolean stdevGradients() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 13));
+    }
+
+    public boolean stdevUpdates() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 14));
+    }
+
+    public boolean stdevActivations() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 15));
+    }
+
+    public boolean meanMagnitudeParameters() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 16));
+    }
+
+    public boolean meanMagnitudeGradients() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 17));
+    }
+
+    public boolean meanMagnitudeUpdates() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 18));
+    }
+
+    public boolean meanMagnitudeActivations() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 19));
+    }
+
+    public boolean learningRatesPresent() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 20));
+    }
+
+    public boolean dataSetMetaDataPresent() {
+        return 0 != (buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN) & (1 << 21));
+    }
+
+    public String toString() {
+        return appendTo(new StringBuilder(100)).toString();
+    }
+
+    public StringBuilder appendTo(final StringBuilder builder) {
+        builder.append('{');
+        boolean atLeastOne = false;
+        if (score()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("score");
+            atLeastOne = true;
+        }
+        if (memoryUse()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("memoryUse");
+            atLeastOne = true;
+        }
+        if (performance()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("performance");
+            atLeastOne = true;
+        }
+        if (garbageCollection()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("garbageCollection");
+            atLeastOne = true;
+        }
+        if (histogramParameters()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("histogramParameters");
+            atLeastOne = true;
+        }
+        if (histogramGradients()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("histogramGradients");
+            atLeastOne = true;
+        }
+        if (histogramUpdates()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("histogramUpdates");
+            atLeastOne = true;
+        }
+        if (histogramActivations()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("histogramActivations");
+            atLeastOne = true;
+        }
+        if (meanParameters()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("meanParameters");
+            atLeastOne = true;
+        }
+        if (meanGradients()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("meanGradients");
+            atLeastOne = true;
+        }
+        if (meanUpdates()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("meanUpdates");
+            atLeastOne = true;
+        }
+        if (meanActivations()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("meanActivations");
+            atLeastOne = true;
+        }
+        if (stdevParameters()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("stdevParameters");
+            atLeastOne = true;
+        }
+        if (stdevGradients()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("stdevGradients");
+            atLeastOne = true;
+        }
+        if (stdevUpdates()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("stdevUpdates");
+            atLeastOne = true;
+        }
+        if (stdevActivations()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("stdevActivations");
+            atLeastOne = true;
+        }
+        if (meanMagnitudeParameters()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("meanMagnitudeParameters");
+            atLeastOne = true;
+        }
+        if (meanMagnitudeGradients()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("meanMagnitudeGradients");
+            atLeastOne = true;
+        }
+        if (meanMagnitudeUpdates()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("meanMagnitudeUpdates");
+            atLeastOne = true;
+        }
+        if (meanMagnitudeActivations()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("meanMagnitudeActivations");
+            atLeastOne = true;
+        }
+        if (learningRatesPresent()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("learningRatesPresent");
+            atLeastOne = true;
+        }
+        if (dataSetMetaDataPresent()) {
+            if (atLeastOne) {
+                builder.append(',');
+            }
+            builder.append("dataSetMetaDataPresent");
+            atLeastOne = true;
+        }
+        builder.append('}');
+
+        return builder;
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateFieldsPresentEncoder.java

@@ -0,0 +1,201 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+import org.agrona.MutableDirectBuffer;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.UpdateFieldsPresentEncoder"})
+@SuppressWarnings("all")
+public class UpdateFieldsPresentEncoder {
+    public static final int ENCODED_LENGTH = 4;
+    private MutableDirectBuffer buffer;
+    private int offset;
+
+    public UpdateFieldsPresentEncoder wrap(final MutableDirectBuffer buffer, final int offset) {
+        this.buffer = buffer;
+        this.offset = offset;
+
+        return this;
+    }
+
+    public int encodedLength() {
+        return ENCODED_LENGTH;
+    }
+
+    public UpdateFieldsPresentEncoder clear() {
+        buffer.putInt(offset, (int) 0L, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder score(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 0) : bits & ~(1 << 0);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder memoryUse(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 1) : bits & ~(1 << 1);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder performance(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 2) : bits & ~(1 << 2);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder garbageCollection(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 3) : bits & ~(1 << 3);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder histogramParameters(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 4) : bits & ~(1 << 4);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder histogramGradients(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 5) : bits & ~(1 << 5);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder histogramUpdates(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 6) : bits & ~(1 << 6);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder histogramActivations(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 7) : bits & ~(1 << 7);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder meanParameters(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 8) : bits & ~(1 << 8);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder meanGradients(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 9) : bits & ~(1 << 9);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder meanUpdates(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 10) : bits & ~(1 << 10);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder meanActivations(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 11) : bits & ~(1 << 11);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder stdevParameters(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 12) : bits & ~(1 << 12);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder stdevGradients(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 13) : bits & ~(1 << 13);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder stdevUpdates(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 14) : bits & ~(1 << 14);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder stdevActivations(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 15) : bits & ~(1 << 15);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder meanMagnitudeParameters(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 16) : bits & ~(1 << 16);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder meanMagnitudeGradients(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 17) : bits & ~(1 << 17);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder meanMagnitudeUpdates(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 18) : bits & ~(1 << 18);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder meanMagnitudeActivations(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 19) : bits & ~(1 << 19);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder learningRatesPresent(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 20) : bits & ~(1 << 20);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+    public UpdateFieldsPresentEncoder dataSetMetaDataPresent(final boolean value) {
+        int bits = buffer.getInt(offset, java.nio.ByteOrder.LITTLE_ENDIAN);
+        bits = value ? bits | (1 << 21) : bits & ~(1 << 21);
+        buffer.putInt(offset, bits, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/VarDataUTF8Decoder.java

@@ -0,0 +1,87 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+import org.agrona.DirectBuffer;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.VarDataUTF8Decoder"})
+@SuppressWarnings("all")
+public class VarDataUTF8Decoder {
+    public static final int ENCODED_LENGTH = -1;
+    private DirectBuffer buffer;
+    private int offset;
+
+    public VarDataUTF8Decoder wrap(final DirectBuffer buffer, final int offset) {
+        this.buffer = buffer;
+        this.offset = offset;
+
+        return this;
+    }
+
+    public int encodedLength() {
+        return ENCODED_LENGTH;
+    }
+
+    public static long lengthNullValue() {
+        return 4294967294L;
+    }
+
+    public static long lengthMinValue() {
+        return 0L;
+    }
+
+    public static long lengthMaxValue() {
+        return 1073741824L;
+    }
+
+    public long length() {
+        return (buffer.getInt(offset + 0, java.nio.ByteOrder.LITTLE_ENDIAN) & 0xFFFF_FFFFL);
+    }
+
+
+    public static short varDataNullValue() {
+        return (short) 255;
+    }
+
+    public static short varDataMinValue() {
+        return (short) 0;
+    }
+
+    public static short varDataMaxValue() {
+        return (short) 254;
+    }
+
+    public String toString() {
+        return appendTo(new StringBuilder(100)).toString();
+    }
+
+    public StringBuilder appendTo(final StringBuilder builder) {
+        builder.append('(');
+        //Token{signal=ENCODING, name='length', description='null', id=-1, version=0, encodedLength=4, offset=0, componentTokenCount=1, encoding=Encoding{presence=REQUIRED, primitiveType=UINT32, byteOrder=LITTLE_ENDIAN, minValue=null, maxValue=1073741824, nullValue=null, constValue=null, characterEncoding='UTF-8', epoch='unix', timeUnit=nanosecond, semanticType='null'}}
+        builder.append("length=");
+        builder.append(length());
+        builder.append('|');
+        //Token{signal=ENCODING, name='varData', description='null', id=-1, version=0, encodedLength=-1, offset=4, componentTokenCount=1, encoding=Encoding{presence=REQUIRED, primitiveType=UINT8, byteOrder=LITTLE_ENDIAN, minValue=null, maxValue=null, nullValue=null, constValue=null, characterEncoding='UTF-8', epoch='unix', timeUnit=nanosecond, semanticType='null'}}
+        builder.append(')');
+
+        return builder;
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/VarDataUTF8Encoder.java

@@ -0,0 +1,83 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.stats.sbe;
+
+import org.agrona.MutableDirectBuffer;
+
+@javax.annotation.Generated(value = {"org.deeplearning4j.ui.stats.sbe.VarDataUTF8Encoder"})
+@SuppressWarnings("all")
+public class VarDataUTF8Encoder {
+    public static final int ENCODED_LENGTH = -1;
+    private MutableDirectBuffer buffer;
+    private int offset;
+
+    public VarDataUTF8Encoder wrap(final MutableDirectBuffer buffer, final int offset) {
+        this.buffer = buffer;
+        this.offset = offset;
+
+        return this;
+    }
+
+    public int encodedLength() {
+        return ENCODED_LENGTH;
+    }
+
+    public static long lengthNullValue() {
+        return 4294967294L;
+    }
+
+    public static long lengthMinValue() {
+        return 0L;
+    }
+
+    public static long lengthMaxValue() {
+        return 1073741824L;
+    }
+
+    public VarDataUTF8Encoder length(final long value) {
+        buffer.putInt(offset + 0, (int) value, java.nio.ByteOrder.LITTLE_ENDIAN);
+        return this;
+    }
+
+
+    public static short varDataNullValue() {
+        return (short) 255;
+    }
+
+    public static short varDataMinValue() {
+        return (short) 0;
+    }
+
+    public static short varDataMaxValue() {
+        return (short) 254;
+    }
+
+    public String toString() {
+        return appendTo(new StringBuilder(100)).toString();
+    }
+
+    public StringBuilder appendTo(final StringBuilder builder) {
+        VarDataUTF8Decoder writer = new VarDataUTF8Decoder();
+        writer.wrap(buffer, offset);
+
+        return writer.appendTo(builder);
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/AgronaPersistable.java

@@ -0,0 +1,33 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.storage;
+
+import org.agrona.DirectBuffer;
+import org.agrona.MutableDirectBuffer;
+import org.deeplearning4j.core.storage.Persistable;
+
+public interface AgronaPersistable extends Persistable {
+
+    void encode(MutableDirectBuffer buffer);
+
+    void decode(DirectBuffer buffer);
+
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/BaseCollectionStatsStorage.java

@@ -0,0 +1,468 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.storage;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import org.deeplearning4j.core.storage.*;
+
+import java.io.Serializable;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+
+public abstract class BaseCollectionStatsStorage implements StatsStorage {
+
+    protected Set<String> sessionIDs;
+    protected Map<SessionTypeId, StorageMetaData> storageMetaData;
+    protected Map<SessionTypeWorkerId, Persistable> staticInfo;
+
+    protected Map<SessionTypeWorkerId, Map<Long, Persistable>> updates = new ConcurrentHashMap<>();
+
+    protected List<StatsStorageListener> listeners = new ArrayList<>();
+
+    protected BaseCollectionStatsStorage() {
+
+    }
+
+    protected abstract Map<Long, Persistable> getUpdateMap(String sessionID, String typeID, String workerID,
+                    boolean createIfRequired);
+
+    //Return any relevant storage events
+    //We want to return these so they can be logged later. Can't be logged immediately, as this may case a race
+    //condition with whatever is receiving the events: i.e., might get the event before the contents are actually
+    //available in the DB
+    protected List<StatsStorageEvent> checkStorageEvents(Persistable p) {
+        if (listeners.isEmpty())
+            return null;
+
+        int count = 0;
+        StatsStorageEvent newSID = null;
+        StatsStorageEvent newTID = null;
+        StatsStorageEvent newWID = null;
+
+        //Is this a new session ID?
+        if (!sessionIDs.contains(p.getSessionID())) {
+            newSID = new StatsStorageEvent(this, StatsStorageListener.EventType.NewSessionID, p.getSessionID(),
+                            p.getTypeID(), p.getWorkerID(), p.getTimeStamp());
+            count++;
+        }
+
+        //Check for new type and worker IDs
+        //TODO probably more efficient way to do this
+        boolean foundTypeId = false;
+        boolean foundWorkerId = false;
+        String typeId = p.getTypeID();
+        String wid = p.getWorkerID();
+        for (SessionTypeId ts : storageMetaData.keySet()) {
+            if (typeId.equals(ts.getTypeID())) {
+                foundTypeId = true;
+                break;
+            }
+        }
+        for (SessionTypeWorkerId stw : staticInfo.keySet()) {
+            if (!foundTypeId && typeId.equals(stw.getTypeID())) {
+                foundTypeId = true;
+            }
+            if (!foundWorkerId && wid.equals(stw.getWorkerID())) {
+                foundWorkerId = true;
+            }
+            if (foundTypeId && foundWorkerId)
+                break;
+        }
+        if (!foundTypeId || !foundWorkerId) {
+            for (SessionTypeWorkerId stw : updates.keySet()) {
+                if (!foundTypeId && typeId.equals(stw.getTypeID())) {
+                    foundTypeId = true;
+                }
+                if (!foundWorkerId && wid.equals(stw.getWorkerID())) {
+                    foundWorkerId = true;
+                }
+                if (foundTypeId && foundWorkerId)
+                    break;
+            }
+        }
+        if (!foundTypeId) {
+            //New type ID
+            newTID = new StatsStorageEvent(this, StatsStorageListener.EventType.NewTypeID, p.getSessionID(),
+                            p.getTypeID(), p.getWorkerID(), p.getTimeStamp());
+            count++;
+        }
+        if (!foundWorkerId) {
+            //New worker ID
+            newWID = new StatsStorageEvent(this, StatsStorageListener.EventType.NewWorkerID, p.getSessionID(),
+                            p.getTypeID(), p.getWorkerID(), p.getTimeStamp());
+            count++;
+        }
+        if (count == 0)
+            return null;
+        List<StatsStorageEvent> sses = new ArrayList<>(count);
+        if (newSID != null)
+            sses.add(newSID);
+        if (newTID != null)
+            sses.add(newTID);
+        if (newWID != null)
+            sses.add(newWID);
+        return sses;
+    }
+
+    protected void notifyListeners(List<StatsStorageEvent> sses) {
+        if (sses == null || sses.isEmpty() || listeners.isEmpty())
+            return;
+        for (StatsStorageListener l : listeners) {
+            for (StatsStorageEvent e : sses) {
+                l.notify(e);
+            }
+        }
+    }
+
+    @Override
+    public List<String> listSessionIDs() {
+        return new ArrayList<>(sessionIDs);
+    }
+
+    @Override
+    public boolean sessionExists(String sessionID) {
+        return sessionIDs.contains(sessionID);
+    }
+
+    @Override
+    public Persistable getStaticInfo(String sessionID, String typeID, String workerID) {
+        SessionTypeWorkerId id = new SessionTypeWorkerId(sessionID, typeID, workerID);
+        return staticInfo.get(id);
+    }
+
+    @Override
+    public List<Persistable> getAllStaticInfos(String sessionID, String typeID) {
+        List<Persistable> out = new ArrayList<>();
+        for (SessionTypeWorkerId key : staticInfo.keySet()) {
+            if (sessionID.equals(key.getSessionID()) && typeID.equals(key.getTypeID())) {
+                out.add(staticInfo.get(key));
+            }
+        }
+        return out;
+    }
+
+    @Override
+    public List<String> listTypeIDsForSession(String sessionID) {
+        Set<String> typeIDs = new HashSet<>();
+        for (SessionTypeId st : storageMetaData.keySet()) {
+            if (!sessionID.equals(st.getSessionID()))
+                continue;
+            typeIDs.add(st.getTypeID());
+        }
+
+        for (SessionTypeWorkerId stw : staticInfo.keySet()) {
+            if (!sessionID.equals(stw.getSessionID()))
+                continue;
+            typeIDs.add(stw.getTypeID());
+        }
+        for (SessionTypeWorkerId stw : updates.keySet()) {
+            if (!sessionID.equals(stw.getSessionID()))
+                continue;
+            typeIDs.add(stw.getTypeID());
+        }
+
+        return new ArrayList<>(typeIDs);
+    }
+
+    @Override
+    public List<String> listWorkerIDsForSession(String sessionID) {
+        List<String> out = new ArrayList<>();
+        for (SessionTypeWorkerId ids : staticInfo.keySet()) {
+            if (sessionID.equals(ids.getSessionID())) {
+                out.add(ids.getWorkerID());
+            }
+        }
+        return out;
+    }
+
+    @Override
+    public List<String> listWorkerIDsForSessionAndType(String sessionID, String typeID) {
+        List<String> out = new ArrayList<>();
+        for (SessionTypeWorkerId ids : staticInfo.keySet()) {
+            if (sessionID.equals(ids.getSessionID()) && typeID.equals(ids.getTypeID())) {
+                out.add(ids.getWorkerID());
+            }
+        }
+        return out;
+    }
+
+    @Override
+    public int getNumUpdateRecordsFor(String sessionID) {
+        int count = 0;
+        for (SessionTypeWorkerId id : updates.keySet()) {
+            if (sessionID.equals(id.getSessionID())) {
+                Map<Long, Persistable> map = updates.get(id);
+                if (map != null)
+                    count += map.size();
+            }
+        }
+        return count;
+    }
+
+    @Override
+    public int getNumUpdateRecordsFor(String sessionID, String typeID, String workerID) {
+        SessionTypeWorkerId id = new SessionTypeWorkerId(sessionID, typeID, workerID);
+        Map<Long, Persistable> map = updates.get(id);
+        if (map != null)
+            return map.size();
+        return 0;
+    }
+
+    @Override
+    public Persistable getLatestUpdate(String sessionID, String typeID, String workerID) {
+        SessionTypeWorkerId id = new SessionTypeWorkerId(sessionID, typeID, workerID);
+        Map<Long, Persistable> map = updates.get(id);
+        if (map == null || map.isEmpty())
+            return null;
+        long maxTime = Long.MIN_VALUE;
+        for (Long l : map.keySet()) {
+            maxTime = Math.max(maxTime, l);
+        }
+        return map.get(maxTime);
+    }
+
+    @Override
+    public Persistable getUpdate(String sessionID, String typeID, String workerID, long timestamp) {
+        SessionTypeWorkerId id = new SessionTypeWorkerId(sessionID, typeID, workerID);
+        Map<Long, Persistable> map = updates.get(id);
+        if (map == null)
+            return null;
+
+        return map.get(timestamp);
+    }
+
+    @Override
+    public List<Persistable> getLatestUpdateAllWorkers(String sessionID, String typeID) {
+        List<Persistable> list = new ArrayList<>();
+
+        for (SessionTypeWorkerId id : updates.keySet()) {
+            if (sessionID.equals(id.getSessionID()) && typeID.equals(id.getTypeID())) {
+                Persistable p = getLatestUpdate(sessionID, typeID, id.workerID);
+                if (p != null) {
+                    list.add(p);
+                }
+            }
+        }
+
+        return list;
+    }
+
+    @Override
+    public List<Persistable> getAllUpdatesAfter(String sessionID, String typeID, String workerID, long timestamp) {
+        List<Persistable> list = new ArrayList<>();
+
+        Map<Long, Persistable> map = getUpdateMap(sessionID, typeID, workerID, false);
+        if (map == null)
+            return list;
+
+        for (Long time : map.keySet()) {
+            if (time > timestamp) {
+                list.add(map.get(time));
+            }
+        }
+
+        Collections.sort(list, new Comparator<Persistable>() {
+            @Override
+            public int compare(Persistable o1, Persistable o2) {
+                return Long.compare(o1.getTimeStamp(), o2.getTimeStamp());
+            }
+        });
+
+        return list;
+    }
+
+    @Override
+    public List<Persistable> getAllUpdatesAfter(String sessionID, String typeID, long timestamp) {
+        List<Persistable> list = new ArrayList<>();
+
+        for (SessionTypeWorkerId stw : staticInfo.keySet()) {
+            if (stw.getSessionID().equals(sessionID) && stw.getTypeID().equals(typeID)) {
+                Map<Long, Persistable> u = updates.get(stw);
+                if (u == null)
+                    continue;
+                for (long l : u.keySet()) {
+                    if (l > timestamp) {
+                        list.add(u.get(l));
+                    }
+                }
+            }
+        }
+
+        //Sort by time stamp
+        Collections.sort(list, new Comparator<Persistable>() {
+            @Override
+            public int compare(Persistable o1, Persistable o2) {
+                return Long.compare(o1.getTimeStamp(), o2.getTimeStamp());
+            }
+        });
+
+        return list;
+    }
+
+    @Override
+    public StorageMetaData getStorageMetaData(String sessionID, String typeID) {
+        return this.storageMetaData.get(new SessionTypeId(sessionID, typeID));
+    }
+
+    @Override
+    public long[] getAllUpdateTimes(String sessionID, String typeID, String workerID) {
+        SessionTypeWorkerId stw = new SessionTypeWorkerId(sessionID, typeID, workerID);
+        Map<Long,Persistable> m = updates.get(stw);
+        if(m == null){
+            return new long[0];
+        }
+
+        long[] ret = new long[m.size()];
+        int i=0;
+        for(Long l : m.keySet()){
+            ret[i++] = l;
+            if(i >= ret.length)
+                break;  //Map "m" can in principle be modified concurrently while iterating here - resulting in an array index exception
+        }
+        Arrays.sort(ret);
+        return ret;
+    }
+
+    @Override
+    public List<Persistable> getUpdates(String sessionID, String typeID, String workerID, long[] timestamps) {
+        SessionTypeWorkerId stw = new SessionTypeWorkerId(sessionID, typeID, workerID);
+        Map<Long,Persistable> m = updates.get(stw);
+        if(m == null){
+            return Collections.emptyList();
+        }
+
+        List<Persistable> ret = new ArrayList<>(timestamps.length);
+        for(long l : timestamps){
+            Persistable p = m.get(l);
+            if(p != null){
+                ret.add(p);
+            }
+        }
+        return ret;
+    }
+
+    // ----- Store new info -----
+
+    @Override
+    public abstract void putStaticInfo(Persistable staticInfo);
+
+    @Override
+    public void putStaticInfo(Collection<? extends Persistable> staticInfo) {
+        for (Persistable p : staticInfo) {
+            putStaticInfo(p);
+        }
+    }
+
+    @Override
+    public abstract void putUpdate(Persistable update);
+
+    @Override
+    public void putUpdate(Collection<? extends Persistable> updates) {
+        for (Persistable p : updates) {
+            putUpdate(p);
+        }
+    }
+
+    @Override
+    public abstract void putStorageMetaData(StorageMetaData storageMetaData);
+
+    @Override
+    public void putStorageMetaData(Collection<? extends StorageMetaData> storageMetaData) {
+        for (StorageMetaData m : storageMetaData) {
+            putStorageMetaData(m);
+        }
+    }
+
+
+    // ----- Listeners -----
+
+    @Override
+    public void registerStatsStorageListener(StatsStorageListener listener) {
+        if (!this.listeners.contains(listener)) {
+            this.listeners.add(listener);
+        }
+    }
+
+    @Override
+    public void deregisterStatsStorageListener(StatsStorageListener listener) {
+        this.listeners.remove(listener);
+    }
+
+    @Override
+    public void removeAllListeners() {
+        this.listeners.clear();
+    }
+
+    @Override
+    public List<StatsStorageListener> getListeners() {
+        return new ArrayList<>(listeners);
+    }
+
+    @Data
+    public static class SessionTypeWorkerId implements Serializable, Comparable<SessionTypeWorkerId> {
+        private final String sessionID;
+        private final String typeID;
+        private final String workerID;
+
+        public SessionTypeWorkerId(String sessionID, String typeID, String workerID) {
+            this.sessionID = sessionID;
+            this.typeID = typeID;
+            this.workerID = workerID;
+        }
+
+        @Override
+        public int compareTo(SessionTypeWorkerId o) {
+            int c = sessionID.compareTo(o.sessionID);
+            if (c != 0)
+                return c;
+            c = typeID.compareTo(o.typeID);
+            if (c != 0)
+                return c;
+            return workerID.compareTo(workerID);
+        }
+
+        @Override
+        public String toString() {
+            return "(" + sessionID + "," + typeID + "," + workerID + ")";
+        }
+    }
+
+    @AllArgsConstructor
+    @Data
+    public static class SessionTypeId implements Serializable, Comparable<SessionTypeId> {
+        private final String sessionID;
+        private final String typeID;
+
+        @Override
+        public int compareTo(SessionTypeId o) {
+            int c = sessionID.compareTo(o.sessionID);
+            if (c != 0)
+                return c;
+            return typeID.compareTo(o.typeID);
+        }
+
+        @Override
+        public String toString() {
+            return "(" + sessionID + "," + typeID + ")";
+        }
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/FileStatsStorage.java

@@ -0,0 +1,40 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.storage;
+
+import org.deeplearning4j.ui.model.storage.mapdb.MapDBStatsStorage;
+
+import java.io.File;
+
+public class FileStatsStorage extends MapDBStatsStorage {
+
+    private final File file;
+
+    public FileStatsStorage(File f) {
+        super(f);
+        this.file = f;
+    }
+
+    @Override
+    public String toString() {
+        return "FileStatsStorage(" + file.getPath() + ")";
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/InMemoryStatsStorage.java

@@ -0,0 +1,135 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.storage;
+
+import org.deeplearning4j.core.storage.Persistable;
+import org.deeplearning4j.core.storage.StatsStorageEvent;
+import org.deeplearning4j.core.storage.StatsStorageListener;
+import org.deeplearning4j.core.storage.StorageMetaData;
+import org.deeplearning4j.ui.model.storage.mapdb.MapDBStatsStorage;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class InMemoryStatsStorage extends BaseCollectionStatsStorage {
+    private final String uid;
+
+    public InMemoryStatsStorage() {
+        super();
+        String str = UUID.randomUUID().toString();
+        uid = str.substring(0, Math.min(str.length(), 8));
+
+        sessionIDs = Collections.synchronizedSet(new HashSet<String>());
+        storageMetaData = new ConcurrentHashMap<>();
+        staticInfo = new ConcurrentHashMap<>();
+    }
+
+
+    @Override
+    protected synchronized Map<Long, Persistable> getUpdateMap(String sessionID, String typeID, String workerID,
+                    boolean createIfRequired) {
+        SessionTypeWorkerId id = new SessionTypeWorkerId(sessionID, typeID, workerID);
+        if (updates.containsKey(id)) {
+            return updates.get(id);
+        }
+        if (!createIfRequired) {
+            return null;
+        }
+        Map<Long, Persistable> updateMap = new ConcurrentHashMap<>();
+        updates.put(id, updateMap);
+        return updateMap;
+    }
+
+    @Override
+    public void putStaticInfo(Persistable staticInfo) {
+        List<StatsStorageEvent> sses = checkStorageEvents(staticInfo);
+        if (!sessionIDs.contains(staticInfo.getSessionID())) {
+            sessionIDs.add(staticInfo.getSessionID());
+        }
+        SessionTypeWorkerId id = new SessionTypeWorkerId(staticInfo.getSessionID(), staticInfo.getTypeID(),
+                        staticInfo.getWorkerID());
+
+        this.staticInfo.put(id, staticInfo);
+        StatsStorageEvent sse = null;
+        if (!listeners.isEmpty())
+            sse = new StatsStorageEvent(this, StatsStorageListener.EventType.PostStaticInfo, staticInfo.getSessionID(),
+                            staticInfo.getTypeID(), staticInfo.getWorkerID(), staticInfo.getTimeStamp());
+        for (StatsStorageListener l : listeners) {
+            l.notify(sse);
+        }
+
+        notifyListeners(sses);
+    }
+
+    @Override
+    public void putUpdate(Persistable update) {
+        List<StatsStorageEvent> sses = checkStorageEvents(update);
+        Map<Long, Persistable> updateMap =
+                        getUpdateMap(update.getSessionID(), update.getTypeID(), update.getWorkerID(), true);
+        updateMap.put(update.getTimeStamp(), update);
+
+        StatsStorageEvent sse = null;
+        if (!listeners.isEmpty())
+            sse = new StatsStorageEvent(this, StatsStorageListener.EventType.PostUpdate, update.getSessionID(),
+                            update.getTypeID(), update.getWorkerID(), update.getTimeStamp());
+        for (StatsStorageListener l : listeners) {
+            l.notify(sse);
+        }
+
+        notifyListeners(sses);
+    }
+
+    @Override
+    public void putStorageMetaData(StorageMetaData storageMetaData) {
+        List<StatsStorageEvent> sses = checkStorageEvents(storageMetaData);
+        SessionTypeId id = new SessionTypeId(storageMetaData.getSessionID(), storageMetaData.getTypeID());
+        this.storageMetaData.put(id, storageMetaData);
+
+        StatsStorageEvent sse = null;
+        if (!listeners.isEmpty())
+            sse = new StatsStorageEvent(this, StatsStorageListener.EventType.PostMetaData,
+                            storageMetaData.getSessionID(), storageMetaData.getTypeID(), storageMetaData.getWorkerID(),
+                            storageMetaData.getTimeStamp());
+        for (StatsStorageListener l : listeners) {
+            l.notify(sse);
+        }
+
+        notifyListeners(sses);
+    }
+
+
+    @Override
+    public void close() throws IOException {
+        //No op
+    }
+
+    @Override
+    public boolean isClosed() {
+        return false;
+    }
+
+
+    @Override
+    public String toString() {
+        return "InMemoryStatsStorage(uid=" + uid + ")";
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/JavaStorageMetaData.java

@@ -0,0 +1,137 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.storage.impl;
+
+import lombok.Data;
+import org.apache.commons.compress.utils.IOUtils;
+import org.deeplearning4j.core.storage.StorageMetaData;
+import org.deeplearning4j.ui.model.stats.impl.SbeUtil;
+
+import java.io.*;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+
+@Data
+public class JavaStorageMetaData implements StorageMetaData {
+
+    private long timeStamp;
+    private String sessionID;
+    private String typeID;
+    private String workerID;
+    private String initTypeClass;
+    private String updateTypeClass;
+    //Store serialized; saves class exceptions if we don't have the right class, and don't care about deserializing
+    // on this machine, right now
+    private byte[] extraMeta;
+
+    public JavaStorageMetaData() {
+        //No arg constructor for serialization/deserialization
+    }
+
+    public JavaStorageMetaData(long timeStamp, String sessionID, String typeID, String workerID, Class<?> initType,
+                    Class<?> updateType) {
+        this(timeStamp, sessionID, typeID, workerID, (initType != null ? initType.getName() : null),
+                        (updateType != null ? updateType.getName() : null));
+    }
+
+    public JavaStorageMetaData(long timeStamp, String sessionID, String typeID, String workerID, String initTypeClass,
+                    String updateTypeClass) {
+        this(timeStamp, sessionID, typeID, workerID, initTypeClass, updateTypeClass, null);
+    }
+
+    public JavaStorageMetaData(long timeStamp, String sessionID, String typeID, String workerID, String initTypeClass,
+                    String updateTypeClass, Serializable extraMetaData) {
+        this.timeStamp = timeStamp;
+        this.sessionID = sessionID;
+        this.typeID = typeID;
+        this.workerID = workerID;
+        this.initTypeClass = initTypeClass;
+        this.updateTypeClass = updateTypeClass;
+        this.extraMeta = (extraMetaData == null ? null : SbeUtil.toBytesSerializable(extraMetaData));
+    }
+
+    @Override
+    public int encodingLengthBytes() {
+        //TODO - presumably a more efficient way to do this
+        byte[] encoded = encode();
+        return encoded.length;
+    }
+
+    @Override
+    public byte[] encode() {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
+            oos.writeObject(this);
+        } catch (IOException e) {
+            throw new RuntimeException(e); //Should never happen
+        }
+        return baos.toByteArray();
+    }
+
+    @Override
+    public void encode(ByteBuffer buffer) {
+        buffer.put(encode());
+    }
+
+    @Override
+    public void encode(OutputStream outputStream) throws IOException {
+        try (ObjectOutputStream oos = new ObjectOutputStream(outputStream)) {
+            oos.writeObject(this);
+        }
+    }
+
+    @Override
+    public void decode(byte[] decode) {
+        JavaStorageMetaData r;
+        try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(decode))) {
+            r = (JavaStorageMetaData) ois.readObject();
+        } catch (IOException | ClassNotFoundException e) {
+            throw new RuntimeException(e); //Should never happen
+        }
+
+        Field[] fields = JavaStorageMetaData.class.getDeclaredFields();
+        for (Field f : fields) {
+            f.setAccessible(true);
+            try {
+                f.set(this, f.get(r));
+            } catch (IllegalAccessException e) {
+                throw new RuntimeException(e); //Should never happen
+            }
+        }
+    }
+
+    @Override
+    public void decode(ByteBuffer buffer) {
+        byte[] bytes = new byte[buffer.remaining()];
+        buffer.get(bytes);
+        decode(bytes);
+    }
+
+    @Override
+    public void decode(InputStream inputStream) throws IOException {
+        decode(IOUtils.toByteArray(inputStream));
+    }
+
+    @Override
+    public Serializable getExtraMetaData() {
+        return null;
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/QueuePairStatsStorageListener.java

@@ -0,0 +1,41 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.storage.impl;
+
+import lombok.AllArgsConstructor;
+import org.deeplearning4j.core.storage.StatsStorage;
+import org.deeplearning4j.core.storage.StatsStorageEvent;
+import org.deeplearning4j.core.storage.StatsStorageListener;
+import org.nd4j.common.primitives.Pair;
+
+import java.util.Queue;
+
+@AllArgsConstructor
+public class QueuePairStatsStorageListener implements StatsStorageListener {
+
+    private final StatsStorage statsStorage;
+    private final Queue<Pair<StatsStorage, StatsStorageEvent>> queue;
+
+    @Override
+    public void notify(StatsStorageEvent event) {
+        queue.add(new Pair<>(statsStorage, event));
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/QueueStatsStorageListener.java

@@ -0,0 +1,38 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.storage.impl;
+
+import lombok.AllArgsConstructor;
+import org.deeplearning4j.core.storage.StatsStorageEvent;
+import org.deeplearning4j.core.storage.StatsStorageListener;
+
+import java.util.Queue;
+
+@AllArgsConstructor
+public class QueueStatsStorageListener implements StatsStorageListener {
+
+    private final Queue<StatsStorageEvent> queue;
+
+    @Override
+    public void notify(StatsStorageEvent event) {
+        queue.add(event);
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/SbeStorageMetaData.java

@@ -0,0 +1,208 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.storage.impl;
+
+import lombok.Data;
+import org.agrona.DirectBuffer;
+import org.agrona.MutableDirectBuffer;
+import org.agrona.concurrent.UnsafeBuffer;
+import org.apache.commons.io.IOUtils;
+import org.deeplearning4j.core.storage.StorageMetaData;
+import org.deeplearning4j.ui.model.stats.impl.SbeUtil;
+import org.deeplearning4j.ui.model.stats.sbe.MessageHeaderDecoder;
+import org.deeplearning4j.ui.model.stats.sbe.MessageHeaderEncoder;
+import org.deeplearning4j.ui.model.stats.sbe.StorageMetaDataDecoder;
+import org.deeplearning4j.ui.model.stats.sbe.StorageMetaDataEncoder;
+import org.deeplearning4j.ui.model.storage.AgronaPersistable;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+
+@Data
+public class SbeStorageMetaData implements StorageMetaData, AgronaPersistable {
+
+
+    private long timeStamp;
+    private String sessionID;
+    private String typeID;
+    private String workerID;
+    private String initTypeClass;
+    private String updateTypeClass;
+    //Store serialized; saves class exceptions if we don't have the right class, and don't care about deserializing
+    // on this machine, right now
+    private byte[] extraMeta;
+
+    public SbeStorageMetaData() {
+        //No arg constructor for serialization/deserialization
+    }
+
+    public SbeStorageMetaData(long timeStamp, String sessionID, String typeID, String workerID, Class<?> initType,
+                    Class<?> updateType) {
+        this(timeStamp, sessionID, typeID, workerID, (initType != null ? initType.getName() : null),
+                        (updateType != null ? updateType.getName() : null));
+    }
+
+    public SbeStorageMetaData(long timeStamp, String sessionID, String typeID, String workerID, String initTypeClass,
+                    String updateTypeClass) {
+        this(timeStamp, sessionID, typeID, workerID, initTypeClass, updateTypeClass, null);
+    }
+
+    public SbeStorageMetaData(long timeStamp, String sessionID, String typeID, String workerID, String initTypeClass,
+                    String updateTypeClass, Serializable extraMetaData) {
+        this.timeStamp = timeStamp;
+        this.sessionID = sessionID;
+        this.typeID = typeID;
+        this.workerID = workerID;
+        this.initTypeClass = initTypeClass;
+        this.updateTypeClass = updateTypeClass;
+        this.extraMeta = (extraMetaData == null ? null : SbeUtil.toBytesSerializable(extraMetaData));
+    }
+
+    public Serializable getExtraMetaData() {
+        return SbeUtil.fromBytesSerializable(extraMeta);
+    }
+
+    @Override
+    public int encodingLengthBytes() {
+        //TODO store byte[]s so we don't end up calculating again in encode
+        //SBE buffer is composed of:
+        //(a) Header: 8 bytes (4x uint16 = 8 bytes)
+        //(b) timestamp: fixed length long value (8 bytes)
+        //(b) 5 variable length fields. 4 bytes header (each) + content = 20 bytes + content
+        //(c) Variable length byte[]. 4 bytes header + content
+
+        int bufferSize = 8 + 8 + 20 + 4;
+        byte[] bSessionID = SbeUtil.toBytes(true, sessionID);
+        byte[] bTypeID = SbeUtil.toBytes(true, typeID);
+        byte[] bWorkerID = SbeUtil.toBytes(true, workerID);
+        byte[] bInitTypeClass = SbeUtil.toBytes(true, initTypeClass);
+        byte[] bUpdateTypeClass = SbeUtil.toBytes(true, updateTypeClass);
+        byte[] bExtraMetaData = SbeUtil.toBytesSerializable(extraMeta);
+
+        bufferSize += bSessionID.length + bTypeID.length + bWorkerID.length + bInitTypeClass.length
+                        + bUpdateTypeClass.length + bExtraMetaData.length;
+
+        return bufferSize;
+    }
+
+    @Override
+    public byte[] encode() {
+        byte[] bytes = new byte[encodingLengthBytes()];
+        MutableDirectBuffer buffer = new UnsafeBuffer(bytes);
+        encode(buffer);
+        return bytes;
+    }
+
+    @Override
+    public void encode(ByteBuffer buffer) {
+        encode(new UnsafeBuffer(buffer));
+    }
+
+    @Override
+    public void encode(MutableDirectBuffer buffer) {
+        MessageHeaderEncoder enc = new MessageHeaderEncoder();
+        StorageMetaDataEncoder smde = new StorageMetaDataEncoder();
+
+        enc.wrap(buffer, 0).blockLength(smde.sbeBlockLength()).templateId(smde.sbeTemplateId())
+                        .schemaId(smde.sbeSchemaId()).version(smde.sbeSchemaVersion());
+
+        int offset = enc.encodedLength(); //Expect 8 bytes
+
+        byte[] bSessionID = SbeUtil.toBytes(true, sessionID);
+        byte[] bTypeID = SbeUtil.toBytes(true, typeID);
+        byte[] bWorkerID = SbeUtil.toBytes(true, workerID);
+        byte[] bInitTypeClass = SbeUtil.toBytes(true, initTypeClass);
+        byte[] bUpdateTypeClass = SbeUtil.toBytes(true, updateTypeClass);
+
+        smde.wrap(buffer, offset).timeStamp(timeStamp);
+
+        StorageMetaDataEncoder.ExtraMetaDataBytesEncoder ext =
+                        smde.extraMetaDataBytesCount(extraMeta == null ? 0 : extraMeta.length);
+        if (extraMeta != null) {
+            for (byte b : extraMeta) {
+                ext.next().bytes(b);
+            }
+        }
+
+        smde.putSessionID(bSessionID, 0, bSessionID.length).putTypeID(bTypeID, 0, bTypeID.length)
+                        .putWorkerID(bWorkerID, 0, bWorkerID.length)
+                        .putInitTypeClass(bInitTypeClass, 0, bInitTypeClass.length)
+                        .putUpdateTypeClass(bUpdateTypeClass, 0, bUpdateTypeClass.length);
+    }
+
+    @Override
+    public void encode(OutputStream outputStream) throws IOException {
+        //TODO there may be more efficient way of doing this
+        outputStream.write(encode());
+    }
+
+    @Override
+    public void decode(byte[] decode) {
+        MutableDirectBuffer buffer = new UnsafeBuffer(decode);
+        decode(buffer);
+    }
+
+    @Override
+    public void decode(ByteBuffer buffer) {
+        decode(new UnsafeBuffer(buffer));
+    }
+
+    @Override
+    public void decode(DirectBuffer buffer) {
+
+        MessageHeaderDecoder dec = new MessageHeaderDecoder();
+        dec.wrap(buffer, 0);
+        final int blockLength = dec.blockLength();
+        final int version = dec.version();
+        final int headerLength = dec.encodedLength();
+
+        //TODO Validate header, version etc
+
+        StorageMetaDataDecoder smdd = new StorageMetaDataDecoder();
+        smdd.wrap(buffer, headerLength, blockLength, version);
+        timeStamp = smdd.timeStamp();
+
+        StorageMetaDataDecoder.ExtraMetaDataBytesDecoder ext = smdd.extraMetaDataBytes();
+        int length = ext.count();
+        if (length > 0) {
+            extraMeta = new byte[length];
+            int i = 0;
+            for (StorageMetaDataDecoder.ExtraMetaDataBytesDecoder d : ext) {
+                extraMeta[i++] = d.bytes();
+            }
+        }
+
+        sessionID = smdd.sessionID();
+        typeID = smdd.typeID();
+        workerID = smdd.workerID();
+        initTypeClass = smdd.initTypeClass();
+        updateTypeClass = smdd.updateTypeClass();
+    }
+
+    @Override
+    public void decode(InputStream inputStream) throws IOException {
+        byte[] bytes = IOUtils.toByteArray(inputStream);
+        decode(bytes);
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/mapdb/MapDBStatsStorage.java

@@ -0,0 +1,344 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.storage.mapdb;
+
+import lombok.Data;
+import lombok.NonNull;
+import org.deeplearning4j.config.DL4JClassLoading;
+import org.deeplearning4j.core.storage.*;
+import org.deeplearning4j.ui.model.storage.BaseCollectionStatsStorage;
+import org.mapdb.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class MapDBStatsStorage extends BaseCollectionStatsStorage {
+
+    private static final String COMPOSITE_KEY_HEADER = "&&&";
+    private static final String COMPOSITE_KEY_SEPARATOR = "@@@";
+
+    private boolean isClosed = false;
+    private DB db;
+    private Lock updateMapLock = new ReentrantLock(true);
+
+    private Map<String, Integer> classToInteger; //For storage
+    private Map<Integer, String> integerToClass; //For storage
+    private Atomic.Integer classCounter;
+
+    public MapDBStatsStorage() {
+        this(new Builder());
+    }
+
+    public MapDBStatsStorage(File f) {
+        this(new Builder().file(f));
+    }
+
+    private MapDBStatsStorage(Builder builder) {
+        File f = builder.getFile();
+
+        if (f == null) {
+            //In-Memory Stats Storage
+            db = DBMaker.memoryDB().make();
+        } else {
+            db = DBMaker.fileDB(f).closeOnJvmShutdown().transactionEnable() //Default to Write Ahead Log - lower performance, but has crash protection
+                            .make();
+        }
+
+        //Initialize/open the required maps/lists
+        sessionIDs = db.hashSet("sessionIDs", Serializer.STRING).createOrOpen();
+        storageMetaData = db.hashMap("storageMetaData").keySerializer(new SessionTypeIdSerializer())
+                        .valueSerializer(new PersistableSerializer<StorageMetaData>()).createOrOpen();
+        staticInfo = db.hashMap("staticInfo").keySerializer(new SessionTypeWorkerIdSerializer())
+                        .valueSerializer(new PersistableSerializer<>()).createOrOpen();
+
+        classToInteger = db.hashMap("classToInteger").keySerializer(Serializer.STRING)
+                        .valueSerializer(Serializer.INTEGER).createOrOpen();
+
+        integerToClass = db.hashMap("integerToClass").keySerializer(Serializer.INTEGER)
+                        .valueSerializer(Serializer.STRING).createOrOpen();
+
+        classCounter = db.atomicInteger("classCounter").createOrOpen();
+
+        //Load up any saved update maps to the update map...
+        for (String s : db.getAllNames()) {
+            if (s.startsWith(COMPOSITE_KEY_HEADER)) {
+                Map<Long, Persistable> m = db.hashMap(s).keySerializer(Serializer.LONG)
+                                .valueSerializer(new PersistableSerializer<>()).open();
+                String[] arr = s.split(COMPOSITE_KEY_SEPARATOR);
+                arr[0] = arr[0].substring(COMPOSITE_KEY_HEADER.length()); //Remove header...
+                SessionTypeWorkerId id = new SessionTypeWorkerId(arr[0], arr[1], arr[2]);
+                updates.put(id, m);
+            }
+        }
+    }
+
+    @Override
+    protected Map<Long, Persistable> getUpdateMap(String sessionID, String typeID, String workerID,
+                    boolean createIfRequired) {
+        SessionTypeWorkerId id = new SessionTypeWorkerId(sessionID, typeID, workerID);
+        if (updates.containsKey(id)) {
+            return updates.get(id);
+        }
+        if (!createIfRequired) {
+            return null;
+        }
+        String compositeKey = COMPOSITE_KEY_HEADER + sessionID + COMPOSITE_KEY_SEPARATOR + typeID
+                        + COMPOSITE_KEY_SEPARATOR + workerID;
+
+        Map<Long, Persistable> updateMap;
+        updateMapLock.lock();
+        try {
+            //Try again, in case another thread created it before lock was acquired in this thread
+            if (updates.containsKey(id)) {
+                return updates.get(id);
+            }
+            updateMap = db.hashMap(compositeKey).keySerializer(Serializer.LONG)
+                            .valueSerializer(new PersistableSerializer<>()).createOrOpen();
+            updates.put(id, updateMap);
+        } finally {
+            updateMapLock.unlock();
+        }
+
+        return updateMap;
+    }
+
+
+
+    @Override
+    public void close() {
+        db.commit(); //For write ahead log: need to ensure that we persist all data to disk...
+        db.close();
+        isClosed = true;
+    }
+
+    @Override
+    public boolean isClosed() {
+        return isClosed;
+    }
+
+    // ----- Store new info -----
+
+    @Override
+    public void putStaticInfo(Persistable staticInfo) {
+        List<StatsStorageEvent> sses = checkStorageEvents(staticInfo);
+        if (!sessionIDs.contains(staticInfo.getSessionID())) {
+            sessionIDs.add(staticInfo.getSessionID());
+        }
+        SessionTypeWorkerId id = new SessionTypeWorkerId(staticInfo.getSessionID(), staticInfo.getTypeID(),
+                        staticInfo.getWorkerID());
+
+        this.staticInfo.put(id, staticInfo);
+        db.commit(); //For write ahead log: need to ensure that we persist all data to disk...
+        StatsStorageEvent sse = null;
+        if (!listeners.isEmpty())
+            sse = new StatsStorageEvent(this, StatsStorageListener.EventType.PostStaticInfo, staticInfo.getSessionID(),
+                            staticInfo.getTypeID(), staticInfo.getWorkerID(), staticInfo.getTimeStamp());
+        for (StatsStorageListener l : listeners) {
+            l.notify(sse);
+        }
+
+        notifyListeners(sses);
+    }
+
+    @Override
+    public void putUpdate(Persistable update) {
+        List<StatsStorageEvent> sses = checkStorageEvents(update);
+        Map<Long, Persistable> updateMap =
+                        getUpdateMap(update.getSessionID(), update.getTypeID(), update.getWorkerID(), true);
+        updateMap.put(update.getTimeStamp(), update);
+        db.commit(); //For write ahead log: need to ensure that we persist all data to disk...
+
+        StatsStorageEvent sse = null;
+        if (!listeners.isEmpty())
+            sse = new StatsStorageEvent(this, StatsStorageListener.EventType.PostUpdate, update.getSessionID(),
+                            update.getTypeID(), update.getWorkerID(), update.getTimeStamp());
+        for (StatsStorageListener l : listeners) {
+            l.notify(sse);
+        }
+
+        notifyListeners(sses);
+    }
+
+    @Override
+    public void putStorageMetaData(StorageMetaData storageMetaData) {
+        List<StatsStorageEvent> sses = checkStorageEvents(storageMetaData);
+        SessionTypeId id = new SessionTypeId(storageMetaData.getSessionID(), storageMetaData.getTypeID());
+        this.storageMetaData.put(id, storageMetaData);
+        db.commit(); //For write ahead log: need to ensure that we persist all data to disk...
+
+        StatsStorageEvent sse = null;
+        if (!listeners.isEmpty())
+            sse = new StatsStorageEvent(this, StatsStorageListener.EventType.PostMetaData,
+                            storageMetaData.getSessionID(), storageMetaData.getTypeID(), storageMetaData.getWorkerID(),
+                            storageMetaData.getTimeStamp());
+        for (StatsStorageListener l : listeners) {
+            l.notify(sse);
+        }
+
+        notifyListeners(sses);
+    }
+
+
+    @Data
+    public static class Builder {
+
+        private File file;
+        private boolean useWriteAheadLog = true;
+
+        public Builder() {
+            this(null);
+        }
+
+        public Builder(File file) {
+            this.file = file;
+        }
+
+        public Builder file(File file) {
+            this.file = file;
+            return this;
+        }
+
+        public Builder useWriteAheadLog(boolean useWriteAheadLog) {
+            this.useWriteAheadLog = useWriteAheadLog;
+            return this;
+        }
+
+        public MapDBStatsStorage build() {
+            return new MapDBStatsStorage(this);
+        }
+
+    }
+
+
+    private int getIntForClass(Class<?> c) {
+        String str = c.getName();
+        if (classToInteger.containsKey(str)) {
+            return classToInteger.get(str);
+        }
+        int idx = classCounter.getAndIncrement();
+        classToInteger.put(str, idx);
+        integerToClass.put(idx, str);
+        db.commit();
+        return idx;
+    }
+
+    private String getClassForInt(int integer) {
+        String c = integerToClass.get(integer);
+        if (c == null)
+            throw new RuntimeException("Unknown class index: " + integer); //Should never happen
+        return c;
+    }
+
+    //Simple serializer, based on MapDB's SerializerJava
+    private static class SessionTypeWorkerIdSerializer implements Serializer<SessionTypeWorkerId> {
+        @Override
+        public void serialize(@NonNull DataOutput2 out, @NonNull SessionTypeWorkerId value) throws IOException {
+            ObjectOutputStream out2 = new ObjectOutputStream(out);
+            out2.writeObject(value);
+            out2.flush();
+        }
+
+        @Override
+        public SessionTypeWorkerId deserialize(@NonNull DataInput2 in, int available) throws IOException {
+            try {
+                ObjectInputStream in2 = new ObjectInputStream(new DataInput2.DataInputToStream(in));
+                return (SessionTypeWorkerId) in2.readObject();
+            } catch (ClassNotFoundException e) {
+                throw new IOException(e);
+            }
+        }
+
+        @Override
+        public int compare(SessionTypeWorkerId w1, SessionTypeWorkerId w2) {
+            return w1.compareTo(w2);
+        }
+    }
+
+    //Simple serializer, based on MapDB's SerializerJava
+    private static class SessionTypeIdSerializer implements Serializer<SessionTypeId> {
+        @Override
+        public void serialize(@NonNull DataOutput2 out, @NonNull SessionTypeId value) throws IOException {
+            ObjectOutputStream out2 = new ObjectOutputStream(out);
+            out2.writeObject(value);
+            out2.flush();
+        }
+
+        @Override
+        public SessionTypeId deserialize(@NonNull DataInput2 in, int available) throws IOException {
+            try {
+                ObjectInputStream in2 = new ObjectInputStream(new DataInput2.DataInputToStream(in));
+                return (SessionTypeId) in2.readObject();
+            } catch (ClassNotFoundException e) {
+                throw new IOException(e);
+            }
+        }
+
+        @Override
+        public int compare(SessionTypeId w1, SessionTypeId w2) {
+            return w1.compareTo(w2);
+        }
+    }
+
+    private class PersistableSerializer<T extends Persistable> implements Serializer<T> {
+
+        @Override
+        public void serialize(@NonNull DataOutput2 out, @NonNull Persistable value) throws IOException {
+            //Persistable values can't be decoded in isolation, i.e., without knowing the type
+            //So, we'll first write an integer representing the class name, so we can decode it later...
+            int classIdx = getIntForClass(value.getClass());
+            out.writeInt(classIdx);
+            value.encode(out);
+        }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        public T deserialize(@NonNull DataInput2 input, int available) throws IOException {
+            int classIdx = input.readInt();
+            String className = getClassForInt(classIdx);
+
+            Persistable persistable = DL4JClassLoading.createNewInstance(className);
+
+            int remainingLength = available - 4; // -4 for int class index
+            byte[] temp = new byte[remainingLength];
+            input.readFully(temp);
+            persistable.decode(temp);
+            return (T) persistable;
+        }
+
+        @Override
+        public int compare(Persistable p1, Persistable p2) {
+            int c = p1.getSessionID().compareTo(p2.getSessionID());
+            if (c != 0)
+                return c;
+            c = p1.getTypeID().compareTo(p2.getTypeID());
+            if (c != 0)
+                return c;
+            return p1.getWorkerID().compareTo(p2.getWorkerID());
+        }
+    }
+
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/sqlite/J7FileStatsStorage.java

@@ -0,0 +1,673 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.storage.sqlite;
+
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import lombok.NonNull;
+import org.deeplearning4j.core.storage.*;
+import org.deeplearning4j.ui.model.storage.FileStatsStorage;
+import org.nd4j.common.primitives.Pair;
+
+import java.io.*;
+import java.sql.*;
+import java.util.*;
+
+public class J7FileStatsStorage implements StatsStorage {
+
+    private static final String TABLE_NAME_METADATA = "StorageMetaData";
+    private static final String TABLE_NAME_STATIC_INFO = "StaticInfo";
+    private static final String TABLE_NAME_UPDATES = "Updates";
+
+    private static final String INSERT_META_SQL = "INSERT OR REPLACE INTO " + TABLE_NAME_METADATA
+                    + " (SessionID, TypeID, ObjectClass, ObjectBytes) VALUES ( ?, ?, ?, ? );";
+    private static final String INSERT_STATIC_SQL = "INSERT OR REPLACE INTO " + TABLE_NAME_STATIC_INFO
+                    + " (SessionID, TypeID, WorkerID, ObjectClass, ObjectBytes) VALUES ( ?, ?, ?, ?, ? );";
+    private static final String INSERT_UPDATE_SQL = "INSERT OR REPLACE INTO " + TABLE_NAME_UPDATES
+                    + " (SessionID, TypeID, WorkerID, Timestamp, ObjectClass, ObjectBytes) VALUES ( ?, ?, ?, ?, ?, ? );";
+
+    private final File file;
+    private final Connection connection;
+    private List<StatsStorageListener> listeners = new ArrayList<>();
+
+    /**
+     * @param file Storage location for the stats
+     */
+    public J7FileStatsStorage(@NonNull File file) {
+        this.file = file;
+
+        try {
+            connection = DriverManager.getConnection("jdbc:sqlite:" + file.getAbsolutePath());
+        } catch (Exception e) {
+            throw new RuntimeException("Error ninializing J7FileStatsStorage instance", e);
+        }
+
+        try {
+            initializeTables();
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void initializeTables() throws SQLException {
+
+        //Need tables for:
+        //(a) Metadata  -> session ID and type ID; class; StorageMetaData as a binary BLOB
+        //(b) Static info -> session ID, type ID, worker ID, persistable class, persistable bytes
+        //(c) Update info -> session ID, type ID, worker ID, timestamp, update class, update bytes
+
+        //First: check if tables exist
+        DatabaseMetaData meta = connection.getMetaData();
+        ResultSet rs = meta.getTables(null, null, "%", null);
+        boolean hasStorageMetaDataTable = false;
+        boolean hasStaticInfoTable = false;
+        boolean hasUpdatesTable = false;
+        while (rs.next()) {
+            //3rd value: table name - http://docs.oracle.com/javase/6/docs/api/java/sql/DatabaseMetaData.html#getTables%28java.lang.String,%20java.lang.String,%20java.lang.String,%20java.lang.String[]%29
+            String name = rs.getString(3);
+            if (TABLE_NAME_METADATA.equals(name))
+                hasStorageMetaDataTable = true;
+            else if (TABLE_NAME_STATIC_INFO.equals(name))
+                hasStaticInfoTable = true;
+            else if (TABLE_NAME_UPDATES.equals(name))
+                hasUpdatesTable = true;
+        }
+
+
+        Statement statement = connection.createStatement();
+
+        if (!hasStorageMetaDataTable) {
+            statement.executeUpdate("CREATE TABLE " + TABLE_NAME_METADATA + " (" + "SessionID TEXT NOT NULL, "
+                            + "TypeID TEXT NOT NULL, " + "ObjectClass TEXT NOT NULL, " + "ObjectBytes BLOB NOT NULL, "
+                            + "PRIMARY KEY ( SessionID, TypeID )" + ");");
+        }
+
+        if (!hasStaticInfoTable) {
+            statement.executeUpdate("CREATE TABLE " + TABLE_NAME_STATIC_INFO + " (" + "SessionID TEXT NOT NULL, "
+                            + "TypeID TEXT NOT NULL, " + "WorkerID TEXT NOT NULL, " + "ObjectClass TEXT NOT NULL, "
+                            + "ObjectBytes BLOB NOT NULL, " + "PRIMARY KEY ( SessionID, TypeID, WorkerID )" + ");");
+        }
+
+        if (!hasUpdatesTable) {
+            statement.executeUpdate("CREATE TABLE " + TABLE_NAME_UPDATES + " (" + "SessionID TEXT NOT NULL, "
+                            + "TypeID TEXT NOT NULL, " + "WorkerID TEXT NOT NULL, " + "Timestamp INTEGER NOT NULL, "
+                            + "ObjectClass TEXT NOT NULL, " + "ObjectBytes BLOB NOT NULL, "
+                            + "PRIMARY KEY ( SessionID, TypeID, WorkerID, Timestamp )" + ");");
+        }
+
+        statement.close();
+
+    }
+
+    private static Pair<String, byte[]> serializeForDB(Object object) {
+        String classStr = object.getClass().getName();
+        try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
+                        ObjectOutputStream oos = new ObjectOutputStream(baos)) {
+            oos.writeObject(object);
+            oos.close();
+            byte[] bytes = baos.toByteArray();
+            return new Pair<>(classStr, bytes);
+        } catch (IOException e) {
+            throw new RuntimeException("Error serializing object for storage", e);
+        }
+    }
+
+    private static <T> T deserialize(byte[] bytes) {
+        try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes))) {
+            return (T) ois.readObject();
+        } catch (IOException | ClassNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private <T> T queryAndGet(String sql, int columnIndex) {
+        try (Statement statement = connection.createStatement()) {
+            ResultSet rs = statement.executeQuery(sql);
+            if (!rs.next())
+                return null;
+            byte[] bytes = rs.getBytes(columnIndex);
+            return deserialize(bytes);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private List<String> selectDistinct(String columnName, boolean queryMeta, boolean queryStatic, boolean queryUpdates,
+                    String conditionColumn, String conditionValue) {
+        Set<String> unique = new HashSet<>();
+
+        try (Statement statement = connection.createStatement()) {
+            if (queryMeta) {
+                queryHelper(statement, querySqlHelper(columnName, TABLE_NAME_METADATA, conditionColumn, conditionValue),
+                                unique);
+            }
+
+            if (queryStatic) {
+                queryHelper(statement,
+                                querySqlHelper(columnName, TABLE_NAME_STATIC_INFO, conditionColumn, conditionValue),
+                                unique);
+            }
+
+            if (queryUpdates) {
+                queryHelper(statement, querySqlHelper(columnName, TABLE_NAME_UPDATES, conditionColumn, conditionValue),
+                                unique);
+            }
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+
+        return new ArrayList<>(unique);
+    }
+
+    private String querySqlHelper(String columnName, String table, String conditionColumn, String conditionValue) {
+        String unique = "SELECT DISTINCT " + columnName + " FROM " + table;
+        if (conditionColumn != null) {
+            unique += " WHERE " + conditionColumn + " = '" + conditionValue + "'";
+        }
+        unique += ";";
+        return unique;
+    }
+
+    private void queryHelper(Statement statement, String q, Set<String> unique) throws SQLException {
+        ResultSet rs = statement.executeQuery(q);
+        while (rs.next()) {
+            String str = rs.getString(1);
+            unique.add(str);
+        }
+    }
+
+    protected List<StatsStorageEvent> checkStorageEvents(Persistable p) {
+        if (listeners.isEmpty())
+            return null;
+
+        StatsStorageEvent newSID = null;
+        StatsStorageEvent newTID = null;
+        StatsStorageEvent newWID = null;
+
+        String sid = p.getSessionID();
+        String tid = p.getTypeID();
+        String wid = p.getWorkerID();
+
+        //Is this a new session ID? type ID? worker ID?
+
+        //This is not the most efficient approach
+        boolean isNewSID = false;
+        boolean isNewTID = false;
+        boolean isNewWID = false;
+        if (!listSessionIDs().contains(sid)) {
+            isNewSID = true;
+            isNewTID = true;
+            isNewWID = true;
+        }
+
+        if (!isNewTID && !listTypeIDsForSession(sid).contains(tid)) {
+            isNewTID = true;
+        }
+
+        if (!isNewWID && !listWorkerIDsForSessionAndType(sid, tid).contains(wid)) {
+            isNewWID = true;
+        }
+
+        if (isNewSID) {
+            newSID = new StatsStorageEvent(this, StatsStorageListener.EventType.NewSessionID, p.getSessionID(),
+                            p.getTypeID(), p.getWorkerID(), p.getTimeStamp());
+        }
+        if (isNewTID) {
+            newTID = new StatsStorageEvent(this, StatsStorageListener.EventType.NewTypeID, p.getSessionID(),
+                            p.getTypeID(), p.getWorkerID(), p.getTimeStamp());
+        }
+        if (isNewWID) {
+            newWID = new StatsStorageEvent(this, StatsStorageListener.EventType.NewWorkerID, p.getSessionID(),
+                            p.getTypeID(), p.getWorkerID(), p.getTimeStamp());
+        }
+
+        if (!isNewSID && !isNewTID && !isNewWID)
+            return null;
+        List<StatsStorageEvent> sses = new ArrayList<>(3);
+        if (newSID != null)
+            sses.add(newSID);
+        if (newTID != null)
+            sses.add(newTID);
+        if (newWID != null)
+            sses.add(newWID);
+        return sses;
+    }
+
+    @Override
+    public void putStorageMetaData(StorageMetaData storageMetaData) {
+        putStorageMetaData(Collections.singletonList(storageMetaData));
+    }
+
+    @Override
+    public void putStorageMetaData(Collection<? extends StorageMetaData> collection) {
+        List<StatsStorageEvent> sses = null;
+        try {
+            PreparedStatement ps = connection.prepareStatement(INSERT_META_SQL);
+
+            for (StorageMetaData storageMetaData : collection) {
+                List<StatsStorageEvent> ssesTemp = checkStorageEvents(storageMetaData);
+                if (ssesTemp != null) {
+                    if (sses == null)
+                        sses = ssesTemp;
+                    else
+                        sses.addAll(ssesTemp);
+                }
+
+                if (!listeners.isEmpty()) {
+                    StatsStorageEvent sse = new StatsStorageEvent(this, StatsStorageListener.EventType.PostMetaData,
+                                    storageMetaData.getSessionID(), storageMetaData.getTypeID(),
+                                    storageMetaData.getWorkerID(), storageMetaData.getTimeStamp());
+                    if (sses == null)
+                        sses = new ArrayList<>();
+                    sses.add(sse);
+                }
+
+
+                //Normally we'd batch these... sqlite has an autocommit feature that messes up batching with .addBatch() and .executeUpdate()
+                Pair<String, byte[]> p = serializeForDB(storageMetaData);
+
+                ps.setString(1, storageMetaData.getSessionID());
+                ps.setString(2, storageMetaData.getTypeID());
+                ps.setString(3, p.getFirst());
+                ps.setObject(4, p.getSecond());
+                ps.executeUpdate();
+            }
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+
+        notifyListeners(sses);
+    }
+
+    @Override
+    public void putStaticInfo(Persistable staticInfo) {
+        putStaticInfo(Collections.singletonList(staticInfo));
+    }
+
+    @Override
+    public void putStaticInfo(Collection<? extends Persistable> collection) {
+        List<StatsStorageEvent> sses = null;
+        try {
+            PreparedStatement ps = connection.prepareStatement(INSERT_STATIC_SQL);
+
+            for (Persistable p : collection) {
+                List<StatsStorageEvent> ssesTemp = checkStorageEvents(p);
+                if (ssesTemp != null) {
+                    if (sses == null)
+                        sses = ssesTemp;
+                    else
+                        sses.addAll(ssesTemp);
+                }
+
+                if (!listeners.isEmpty()) {
+                    StatsStorageEvent sse = new StatsStorageEvent(this, StatsStorageListener.EventType.PostStaticInfo,
+                                    p.getSessionID(), p.getTypeID(), p.getWorkerID(), p.getTimeStamp());
+                    if (sses == null)
+                        sses = new ArrayList<>();
+                    sses.add(sse);
+                }
+
+                //Normally we'd batch these... sqlite has an autocommit feature that messes up batching with .addBatch() and .executeUpdate()
+                Pair<String, byte[]> pair = serializeForDB(p);
+
+                ps.setString(1, p.getSessionID());
+                ps.setString(2, p.getTypeID());
+                ps.setString(3, p.getWorkerID());
+                ps.setString(4, pair.getFirst());
+                ps.setBytes(5, pair.getSecond());
+                ps.executeUpdate();
+            }
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+
+        notifyListeners(sses);
+    }
+
+    @Override
+    public void putUpdate(Persistable update) {
+        putUpdate(Collections.singletonList(update));
+    }
+
+    @Override
+    public void putUpdate(Collection<? extends Persistable> collection) {
+        List<StatsStorageEvent> sses = null;
+
+        try {
+            PreparedStatement ps = connection.prepareStatement(INSERT_UPDATE_SQL);
+
+            for (Persistable p : collection) {
+                List<StatsStorageEvent> ssesTemp = checkStorageEvents(p);
+                if (ssesTemp != null) {
+                    if (sses == null)
+                        sses = ssesTemp;
+                    else
+                        sses.addAll(ssesTemp);
+                }
+
+                if (!listeners.isEmpty()) {
+                    StatsStorageEvent sse = new StatsStorageEvent(this, StatsStorageListener.EventType.PostUpdate,
+                                    p.getSessionID(), p.getTypeID(), p.getWorkerID(), p.getTimeStamp());
+                    if (sses == null)
+                        sses = new ArrayList<>();
+                    sses.add(sse);
+                }
+
+                //Normally we'd batch these... sqlite has an autocommit feature that messes up batching with .addBatch() and .executeUpdate()
+                Pair<String, byte[]> pair = serializeForDB(p);
+
+                ps.setString(1, p.getSessionID());
+                ps.setString(2, p.getTypeID());
+                ps.setString(3, p.getWorkerID());
+                ps.setLong(4, p.getTimeStamp());
+                ps.setString(5, pair.getFirst());
+                ps.setObject(6, pair.getSecond());
+                ps.executeUpdate();
+            }
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+
+        notifyListeners(sses);
+    }
+
+    @Override
+    public void close() throws IOException {
+        try {
+            connection.close();
+        } catch (Exception e) {
+            throw new IOException(e);
+        }
+    }
+
+    @Override
+    public boolean isClosed() {
+        try {
+            return connection.isClosed();
+        } catch (Exception e) {
+            return true;
+        }
+    }
+
+    @Override
+    public List<String> listSessionIDs() {
+        return selectDistinct("SessionID", true, true, false, null, null);
+    }
+
+    @Override
+    public boolean sessionExists(String sessionID) {
+        String existsMetaSQL = "SELECT 1 FROM " + TABLE_NAME_METADATA + " WHERE SessionID = '" + sessionID + "';";
+        String existsStaticSQL = "SELECT 1 FROM " + TABLE_NAME_STATIC_INFO + " WHERE SessionID = '" + sessionID + "';";
+
+        try (Statement statement = connection.createStatement()) {
+            ResultSet rs = statement.executeQuery(existsMetaSQL);
+            if (rs.next()) {
+                return true;
+            }
+
+            rs = statement.executeQuery(existsStaticSQL);
+            return rs.next();
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public Persistable getStaticInfo(String sessionID, String typeID, String workerID) {
+        String selectStaticSQL = "SELECT ObjectBytes FROM " + TABLE_NAME_STATIC_INFO + " WHERE SessionID = '"
+                        + sessionID + "' AND TypeID = '" + typeID + "' AND WorkerID = '" + workerID + "';";
+        return queryAndGet(selectStaticSQL, 1);
+    }
+
+    @Override
+    public List<Persistable> getAllStaticInfos(String sessionID, String typeID) {
+        String selectStaticSQL = "SELECT * FROM " + TABLE_NAME_STATIC_INFO + " WHERE SessionID = '" + sessionID
+                        + "' AND TypeID = '" + typeID + "';";
+        try (Statement statement = connection.createStatement()) {
+            ResultSet rs = statement.executeQuery(selectStaticSQL);
+            List<Persistable> out = new ArrayList<>();
+            while (rs.next()) {
+                byte[] bytes = rs.getBytes(5);
+                out.add((Persistable) deserialize(bytes));
+            }
+            return out;
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public List<String> listTypeIDsForSession(String sessionID) {
+        return selectDistinct("TypeID", true, true, true, "SessionID", sessionID);
+    }
+
+    @Override
+    public List<String> listWorkerIDsForSession(String sessionID) {
+        return selectDistinct("WorkerID", false, true, true, "SessionID", sessionID);
+    }
+
+    @Override
+    public List<String> listWorkerIDsForSessionAndType(String sessionID, String typeID) {
+        String uniqueStatic = "SELECT DISTINCT WorkerID FROM " + TABLE_NAME_STATIC_INFO + " WHERE SessionID = '"
+                        + sessionID + "' AND TypeID = '" + typeID + "';";
+        String uniqueUpdates = "SELECT DISTINCT WorkerID FROM " + TABLE_NAME_UPDATES + " WHERE SessionID = '"
+                        + sessionID + "' AND TypeID = '" + typeID + "';";
+
+        Set<String> unique = new HashSet<>();
+        try (Statement statement = connection.createStatement()) {
+            ResultSet rs = statement.executeQuery(uniqueStatic);
+            while (rs.next()) {
+                String str = rs.getString(1);
+                unique.add(str);
+            }
+
+            rs = statement.executeQuery(uniqueUpdates);
+            while (rs.next()) {
+                String str = rs.getString(1);
+                unique.add(str);
+            }
+
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+
+        return new ArrayList<>(unique);
+    }
+
+    @Override
+    public int getNumUpdateRecordsFor(String sessionID) {
+        String sql = "SELECT COUNT(*) FROM " + TABLE_NAME_UPDATES + " WHERE SessionID = '" + sessionID + "';";
+        try (Statement statement = connection.createStatement()) {
+            return statement.executeQuery(sql).getInt(1);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public int getNumUpdateRecordsFor(String sessionID, String typeID, String workerID) {
+        String sql = "SELECT COUNT(*) FROM " + TABLE_NAME_UPDATES + " WHERE SessionID = '" + sessionID
+                        + "' AND TypeID = '" + typeID + "' AND WorkerID = '" + workerID + "';";
+        try (Statement statement = connection.createStatement()) {
+            return statement.executeQuery(sql).getInt(1);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public Persistable getLatestUpdate(String sessionID, String typeID, String workerID) {
+        String sql = "SELECT ObjectBytes FROM " + TABLE_NAME_UPDATES + " WHERE SessionID = '" + sessionID
+                        + "' AND TypeID = '" + typeID + "' AND WorkerID = '" + workerID
+                        + "' ORDER BY Timestamp DESC LIMIT 1;";
+        return queryAndGet(sql, 1);
+    }
+
+    @Override
+    public Persistable getUpdate(String sessionID, String typeId, String workerID, long timestamp) {
+        String sql = "SELECT ObjectBytes FROM " + TABLE_NAME_UPDATES + " WHERE SessionID = '" + sessionID
+                        + "' AND TypeID = '" + typeId + "' AND WorkerID = '" + workerID + "' AND Timestamp = '"
+                        + timestamp + "';";
+        return queryAndGet(sql, 1);
+    }
+
+    @Override
+    public List<Persistable> getLatestUpdateAllWorkers(String sessionID, String typeID) {
+        String sql = "SELECT workerId, MAX(Timestamp) FROM " + TABLE_NAME_UPDATES + " WHERE SessionID ='"
+                + sessionID + "' AND " + "TypeID = '" + typeID + "' GROUP BY workerId";
+
+        Map<String,Long> m = new HashMap<>();
+        try (Statement statement = connection.createStatement()) {
+            ResultSet rs = statement.executeQuery(sql);
+            while (rs.next()) {
+                String wid = rs.getString(1);
+                long ts = rs.getLong(2);
+                m.put(wid, ts);
+            }
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+
+        List<Persistable> out = new ArrayList<>();
+        for(String s : m.keySet()){
+            out.add(getUpdate(sessionID, typeID, s, m.get(s)));
+        }
+        return out;
+    }
+
+    @Override
+    public List<Persistable> getAllUpdatesAfter(String sessionID, String typeID, String workerID, long timestamp) {
+        String sql = "SELECT * FROM " + TABLE_NAME_UPDATES + " WHERE SessionID = '" + sessionID + "' AND TypeID = '"
+                        + typeID + "' AND workerId = '" + workerID + "' AND Timestamp > " + timestamp + ";";
+        try (Statement statement = connection.createStatement()) {
+            ResultSet rs = statement.executeQuery(sql);
+            List<Persistable> out = new ArrayList<>();
+            while (rs.next()) {
+                byte[] bytes = rs.getBytes(6);
+                out.add((Persistable) deserialize(bytes));
+            }
+            return out;
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public List<Persistable> getAllUpdatesAfter(String sessionID, String typeID, long timestamp) {
+        String sql = "SELECT ObjectBytes FROM " + TABLE_NAME_UPDATES + " WHERE SessionID = '" + sessionID + "'  "
+                        + "AND TypeID = '" + typeID + "' AND Timestamp > " + timestamp + ";";
+        return queryUpdates(sql);
+    }
+
+    @Override
+    public long[] getAllUpdateTimes(String sessionID, String typeID, String workerID) {
+        String sql = "SELECT Timestamp FROM " + TABLE_NAME_UPDATES + " WHERE SessionID = '" + sessionID + "'  "
+                + "AND TypeID = '" + typeID + "' AND workerID = '" + workerID + "';";
+        try (Statement statement = connection.createStatement()) {
+            ResultSet rs = statement.executeQuery(sql);
+            LongArrayList list = new LongArrayList();
+            while (rs.next()) {
+                list.add(rs.getLong(1));
+            }
+            return list.toLongArray();
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public List<Persistable> getUpdates(String sessionID, String typeID, String workerID, long[] timestamps) {
+        if(timestamps == null || timestamps.length == 0){
+            return Collections.emptyList();
+        }
+
+        StringBuilder sb = new StringBuilder();
+        sb.append("SELECT ObjectBytes FROM ").append(TABLE_NAME_UPDATES).append(" WHERE SessionID = '").append(sessionID)
+                .append("' AND TypeID = '").append(typeID).append("' AND workerID='").append(workerID)
+                .append("'  AND Timestamp IN (");
+
+        for( int i=0; i<timestamps.length; i++ ){
+            if(i > 0){
+                sb.append(",");
+            }
+            sb.append(timestamps[i]);
+        }
+        sb.append(");");
+
+        String sql = sb.toString();
+        return queryUpdates(sql);
+    }
+
+    private List<Persistable> queryUpdates(String sql){
+        try (Statement statement = connection.createStatement()) {
+            ResultSet rs = statement.executeQuery(sql);
+            List<Persistable> out = new ArrayList<>();
+            while (rs.next()) {
+                byte[] bytes = rs.getBytes(1);
+                out.add((Persistable) deserialize(bytes));
+            }
+            return out;
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public StorageMetaData getStorageMetaData(String sessionID, String typeID) {
+        String sql = "SELECT ObjectBytes FROM " + TABLE_NAME_METADATA + " WHERE SessionID = '" + sessionID
+                        + "' AND TypeID = '" + typeID + "' LIMIT 1;";
+        return queryAndGet(sql, 1);
+    }
+
+    @Override
+    public void registerStatsStorageListener(StatsStorageListener listener) {
+        listeners.add(listener);
+    }
+
+    @Override
+    public void deregisterStatsStorageListener(StatsStorageListener listener) {
+        listeners.remove(listener);
+    }
+
+    @Override
+    public void removeAllListeners() {
+        listeners.clear();
+    }
+
+    @Override
+    public List<StatsStorageListener> getListeners() {
+        return new ArrayList<>(listeners);
+    }
+
+    @Override
+    public String toString() {
+        return "J7FileStatsStorage(file=" + file + ")";
+    }
+
+    protected void notifyListeners(List<StatsStorageEvent> sses) {
+        if (sses == null || sses.isEmpty() || listeners.isEmpty())
+            return;
+        for (StatsStorageListener l : listeners) {
+            for (StatsStorageEvent e : sses) {
+                l.notify(e);
+            }
+        }
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/ConvolutionListenerPersistable.java

@@ -0,0 +1,129 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.weights;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import org.apache.commons.compress.utils.IOUtils;
+import org.deeplearning4j.core.storage.Persistable;
+
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.*;
+import java.nio.ByteBuffer;
+
+@AllArgsConstructor
+@Data
+public class ConvolutionListenerPersistable implements Persistable {
+
+    private static final String TYPE_ID = "ConvolutionalListener";
+
+    private String sessionID;
+    private String workerID;
+    private long timestamp;
+    private transient BufferedImage img;
+
+    public ConvolutionListenerPersistable() {}
+
+    @Override
+    public String getSessionID() {
+        return sessionID;
+    }
+
+    @Override
+    public String getTypeID() {
+        return TYPE_ID;
+    }
+
+    @Override
+    public String getWorkerID() {
+        return workerID;
+    }
+
+    @Override
+    public long getTimeStamp() {
+        return timestamp;
+    }
+
+    @Override
+    public int encodingLengthBytes() {
+        return 0;
+    }
+
+    @Override
+    public byte[] encode() {
+        //Not the most efficient: but it's easy to implement...
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
+            oos.writeObject(this);
+        } catch (IOException e) {
+            throw new RuntimeException(e); //Shouldn't normally happen
+        }
+
+        return baos.toByteArray();
+    }
+
+    @Override
+    public void encode(ByteBuffer buffer) {
+        buffer.put(encode());
+    }
+
+    @Override
+    public void encode(OutputStream outputStream) throws IOException {
+        outputStream.write(encode());
+    }
+
+    @Override
+    public void decode(byte[] decode) {
+        try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(decode))) {
+            ConvolutionListenerPersistable p = (ConvolutionListenerPersistable) ois.readObject();
+            this.sessionID = p.sessionID;
+            this.workerID = p.workerID;
+            this.timestamp = p.getTimeStamp();
+            this.img = p.img;
+        } catch (IOException | ClassNotFoundException e) {
+            throw new RuntimeException(e); //Shouldn't normally happen
+        }
+    }
+
+    @Override
+    public void decode(ByteBuffer buffer) {
+        byte[] arr = new byte[buffer.remaining()];
+        buffer.get(arr);
+        decode(arr);
+    }
+
+    @Override
+    public void decode(InputStream inputStream) throws IOException {
+        byte[] b = IOUtils.toByteArray(inputStream);
+        decode(b);
+    }
+
+    private void writeObject(ObjectOutputStream oos) throws IOException {
+        oos.defaultWriteObject();
+        ImageIO.write(img, "png", oos);
+    }
+
+    private void readObject(ObjectInputStream ois) throws IOException, ClassNotFoundException {
+        ois.defaultReadObject();
+        img = ImageIO.read(ois);
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/HistogramBin.java

@@ -0,0 +1,162 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.weights;
+
+import lombok.Data;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.shade.jackson.annotation.JsonIgnore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+@Data
+public class HistogramBin implements Serializable {
+    private transient INDArray sourceArray;
+    private int numberOfBins;
+    private int rounds;
+    private transient INDArray bins;
+    private double max;
+    private double min;
+    private Map<BigDecimal, AtomicInteger> data = new LinkedHashMap<>();
+
+    private static final Logger log = LoggerFactory.getLogger(HistogramBin.class);
+
+    /**
+     * No-Args constructor should be used only for serialization/deserialization purposes.
+     * In all other cases please use Histogram.Builder()
+     */
+    public HistogramBin() {
+
+    }
+
+    /**
+     * Builds histogram bin for specified array
+     * @param array
+     */
+    public HistogramBin(INDArray array) {
+
+    }
+
+    @JsonIgnore
+    private synchronized void calcHistogram() {
+        max = sourceArray.maxNumber().doubleValue();
+        min = sourceArray.minNumber().doubleValue();
+
+        // TODO: there's probably better way to get around of possible NaNs in max/min
+        if (Double.isInfinite(max))
+            max = Float.MAX_VALUE;
+
+        if (Double.isNaN(max))
+            max = Float.MIN_VALUE;
+
+        if (Double.isInfinite(min))
+            min = Float.MAX_VALUE;
+
+        if (Double.isNaN(min))
+            min = Float.MIN_VALUE;
+
+        bins = Nd4j.create(numberOfBins);
+        final double binSize = (max - min) / (numberOfBins - 1);
+
+
+        data = new LinkedHashMap<>();
+        BigDecimal[] keys = new BigDecimal[numberOfBins];
+
+        for (int x = 0; x < numberOfBins; x++) {
+            BigDecimal pos = new BigDecimal((min + (x * binSize))).setScale(rounds, BigDecimal.ROUND_CEILING);
+            data.put(pos, new AtomicInteger(0));
+            keys[x] = pos;
+        }
+
+        for (int x = 0; x < sourceArray.length(); x++) {
+            double d = sourceArray.getDouble(x);
+            int bin = (int) ((d - min) / binSize);
+
+            if (bin < 0) {
+                bins.putScalar(0, bins.getDouble(0) + 1);
+                data.get(keys[0]).incrementAndGet();
+            } else if (bin >= numberOfBins) {
+                bins.putScalar(numberOfBins - 1, bins.getDouble(numberOfBins - 1) + 1);
+                data.get(keys[numberOfBins - 1]).incrementAndGet();
+            } else {
+                bins.putScalar(bin, bins.getDouble(bin) + 1);
+                data.get(keys[bin]).incrementAndGet();
+            }
+        }
+    }
+
+    public static class Builder {
+        private INDArray source;
+        private int binCount;
+        private int rounds = 2;
+
+        /**
+         * Build Histogram Builder instance for specified array
+         * @param array
+         */
+        public Builder(INDArray array) {
+            this.source = array;
+        }
+
+        /**
+         * Sets number of numbers behind decimal part
+         *
+         * @param rounds
+         * @return
+         */
+        public Builder setRounding(int rounds) {
+            this.rounds = rounds;
+            return this;
+        }
+
+        /**
+         * Specifies number of bins for output histogram
+         *
+         * @param bins
+         * @return
+         */
+        public Builder setBinCount(int bins) {
+            this.binCount = bins;
+            return this;
+        }
+
+        /**
+         * Returns ready-to-use Histogram instance
+         * @return
+         */
+        public HistogramBin build() {
+            HistogramBin histogram = new HistogramBin();
+            histogram.sourceArray = this.source;
+            histogram.numberOfBins = this.binCount;
+            histogram.rounds = this.rounds;
+
+            histogram.calcHistogram();
+
+            return histogram;
+        }
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/beans/CompactModelAndGradient.java

@@ -0,0 +1,154 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.deeplearning4j.ui.model.weights.beans;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Slightly modified version of ModelAndGradient, with binned params/gradients, suitable for fast network transfers for HistogramIterationListener
+ *
+ * @author Adam Gibson
+ */
+
+public class CompactModelAndGradient implements Serializable {
+    private long lastUpdateTime = -1L;
+    private Map<String, Map> parameters;
+    private Map<String, Map> gradients;
+    private double score;
+    private List<Double> scores = new ArrayList<>();
+    private List<Map<String, List<Double>>> updateMagnitudes = new ArrayList<>();
+    private List<Map<String, List<Double>>> paramMagnitudes = new ArrayList<>();
+    private List<String> layerNames = new ArrayList<>();
+    private String path;
+
+
+    public CompactModelAndGradient() {
+        parameters = new HashMap<>();
+        gradients = new HashMap<>();
+    }
+
+
+    public void setLastUpdateTime(long lastUpdateTime) {
+        this.lastUpdateTime = lastUpdateTime;
+    }
+
+    public long getLastUpdateTime() {
+        return lastUpdateTime;
+    }
+
+    public double getScore() {
+        return score;
+    }
+
+    public void setScore(double score) {
+        this.score = score;
+    }
+
+
+    public Map<String, Map> getParameters() {
+        return parameters;
+    }
+
+    public void setParameters(Map<String, Map> parameters) {
+        this.parameters = parameters;
+    }
+
+
+    public Map<String, Map> getGradients() {
+        return gradients;
+    }
+
+    public void setGradients(Map<String, Map> gradients) {
+        this.gradients = gradients;
+    }
+
+    public void setScores(List<Double> scores) {
+        this.scores = scores;
+    }
+
+    public void setPath(String path) {
+        this.path = path;
+    }
+
+    public String getPath() {
+        return path;
+    }
+
+    public List<Double> getScores() {
+        return scores;
+    }
+
+    public void setUpdateMagnitudes(List<Map<String, List<Double>>> updateMagnitudes) {
+        this.updateMagnitudes = updateMagnitudes;
+    }
+
+    public List<Map<String, List<Double>>> getUpdateMagnitudes() {
+        return updateMagnitudes;
+    }
+
+    public void setParamMagnitudes(List<Map<String, List<Double>>> paramMagnitudes) {
+        this.paramMagnitudes = paramMagnitudes;
+    }
+
+    public List<Map<String, List<Double>>> getParamMagnitudes() {
+        return paramMagnitudes;
+    }
+
+    public void setLayerNames(List<String> layerNames) {
+        this.layerNames = layerNames;
+    }
+
+    public List<String> getLayerNames() {
+        return layerNames;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        CompactModelAndGradient that = (CompactModelAndGradient) o;
+
+        if (Double.compare(that.score, score) != 0)
+            return false;
+        if (parameters != null ? !parameters.equals(that.parameters) : that.parameters != null)
+            return false;
+        return !(gradients != null ? !gradients.equals(that.gradients) : that.gradients != null);
+    }
+
+    @Override
+    public int hashCode() {
+        int result;
+        long temp;
+        result = parameters != null ? parameters.hashCode() : 0;
+        result = 31 * result + (gradients != null ? gradients.hashCode() : 0);
+        temp = Double.doubleToLongBits(score);
+        result = 31 * result + (int) (temp ^ (temp >>> 32));
+        return result;
+    }
+}

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/resources/StatsListenerSchemas.xml

@@ -0,0 +1,258 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+  ~ /* ******************************************************************************
+  ~  *
+  ~  *
+  ~  * This program and the accompanying materials are made available under the
+  ~  * terms of the Apache License, Version 2.0 which is available at
+  ~  * https://www.apache.org/licenses/LICENSE-2.0.
+  ~  *
+  ~  *  See the NOTICE file distributed with this work for additional
+  ~  *  information regarding copyright ownership.
+  ~  * Unless required by applicable law or agreed to in writing, software
+  ~  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  ~  * License for the specific language governing permissions and limitations
+  ~  * under the License.
+  ~  *
+  ~  * SPDX-License-Identifier: Apache-2.0
+  ~  ******************************************************************************/
+  -->
+
+<sbe:messageSchema xmlns:sbe="http://fixprotocol.io/2016/sbe"
+                   package="org.deeplearning4j.ui.stats.sbe"
+                   id="1"
+                   version="0"
+                   semanticVersion="0.6.1"
+                   description="Deeplearning4j Stats Listener: Static Information and Updates"
+                   byteOrder="littleEndian">
+    <!--
+    Schemas for encoding information for the stats listener using SBE - Simple Binary Encoding.
+    Two schemas are defined here:
+    (a) StaticInfo - once-off information, passed once upon initialization
+    (b) Update - posted periodically
+    (c) Meta data - type information, for use with the Java SbeStorageMetaData class
+
+    SBE uses code generation to create high-performance binary encoding/decoding classes. These are generated directly
+    from this XML data schema (for Java or C++), using the SBE Java tool.
+    The generated code files are then used in the project, as normal Java files (requires only Agrona as a dependency)
+
+    To build the encoder/decoder files:
+    1. Download sbe-all-1.5.1.jar (later versions should be fine) - http://repo1.maven.org/maven2/uk/co/real-logic/sbe-all/1.5.1/
+    2. Change directory to the directory containing the above jar
+    3. Run the following:
+       java -Dsbe.output.dir=<path>/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/ -jar sbe-all-1.5.1.jar <path>/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/resources/StatsListenerSchemas.xml
+       Where <path> is the path to the DL4J project root directory
+       The "package" property above (org.deeplearning4j...) defines the rest of the output path relative to the output directory
+
+    Some key points to note with SBE:
+    - There are specific rules regarding the formats and ordering for the Schema. You need to understand these before
+      messing with this file, or generating new SBE formats
+    - Encoding and decoding MUST follow the order defined in the schema exactly. Not doing this can result in corrupted
+      data being read. Consequently, to decode any part of the message, you must decode everything that came before it.
+
+    +++ References +++
+    Overview, performance:      http://mechanical-sympathy.blogspot.com/2014/05/simple-binary-encoding.html
+    Wiki:                       https://github.com/real-logic/simple-binary-encoding/wiki
+    SBE tool:                   https://github.com/real-logic/simple-binary-encoding/wiki/Sbe-Tool-Guide
+    SBE tool via Maven:         https://github.com/real-logic/simple-binary-encoding/wiki/Sbe-Tool-Maven
+    Schema:                     https://github.com/real-logic/simple-binary-encoding/wiki/FIX-SBE-XML-Primer
+    Using the generated code:   https://github.com/real-logic/simple-binary-encoding/wiki/Java-Users-Guide
+    Example:                    https://github.com/real-logic/simple-binary-encoding/blob/master/sbe-samples/src/main/java/uk/co/real_logic/sbe/examples/ExampleUsingGeneratedStub.java
+
+    Author: Alex Black
+    -->
+    <types>
+        <composite name="messageHeader" description="Message identifiers and length of message root">
+            <type name="blockLength" primitiveType="uint16"/>
+            <type name="templateId" primitiveType="uint16"/>
+            <type name="schemaId" primitiveType="uint16"/>
+            <type name="version" primitiveType="uint16"/>
+        </composite>
+        <composite name="VarDataUTF8">
+            <type name="length" primitiveType="uint32" maxValue="1073741824"/>
+            <type name="varData" primitiveType="uint8" length="0" characterEncoding="UTF-8"/>
+        </composite>
+        <composite name="groupSizeEncoding" description="Repeating group dimensions">
+            <type name="blockLength" primitiveType="uint16"/>
+            <type name="numInGroup" primitiveType="uint16"/>
+        </composite>
+    </types>
+    <types>
+        <!-- Initialization: Set for encoding presence/absence of software, hardware and model information -->
+        <set name="InitFieldsPresent" encodingType="uint8" semanticType="FieldsPresent">
+            <choice name="softwareInfo">0</choice>
+            <choice name="hardwareInfo">1</choice>
+            <choice name="modelInfo">2</choice>
+        </set>
+
+        <!-- Updates: Set for encoding present/absence of various entries -->
+        <set name="UpdateFieldsPresent" encodingType="uint32" semanticType="UpdateFieldsPresent">
+            <choice name="score">0</choice>
+            <choice name="memoryUse">1</choice>
+            <choice name="performance">2</choice>
+            <choice name="garbageCollection">3</choice>
+            <choice name="histogramParameters">4</choice>
+            <choice name="histogramGradients">5</choice>
+            <choice name="histogramUpdates">6</choice>
+            <choice name="histogramActivations">7</choice>
+            <choice name="meanParameters">8</choice>
+            <choice name="meanGradients">9</choice>
+            <choice name="meanUpdates">10</choice>
+            <choice name="meanActivations">11</choice>
+            <choice name="stdevParameters">12</choice>
+            <choice name="stdevGradients">13</choice>
+            <choice name="stdevUpdates">14</choice>
+            <choice name="stdevActivations">15</choice>
+            <choice name="meanMagnitudeParameters">16</choice>
+            <choice name="meanMagnitudeGradients">17</choice>
+            <choice name="meanMagnitudeUpdates">18</choice>
+            <choice name="meanMagnitudeActivations">19</choice>
+            <choice name="learningRatesPresent">20</choice>
+            <choice name="dataSetMetaDataPresent">21</choice>
+        </set>
+        <enum name="MemoryType" encodingType="uint8">
+            <validValue name="JvmCurrent">0</validValue>
+            <validValue name="JvmMax">1</validValue>
+            <validValue name="OffHeapCurrent">2</validValue>
+            <validValue name="OffHeapMax">3</validValue>
+            <validValue name="DeviceCurrent">4</validValue>
+            <validValue name="DeviceMax">5</validValue>
+        </enum>
+
+        <enum name="StatsType" encodingType="uint8">
+            <validValue name="Parameters">0</validValue>
+            <validValue name="Gradients">1</validValue>
+            <validValue name="Updates">2</validValue>
+            <validValue name="Activations">3</validValue>
+        </enum>
+        <enum name="SummaryType" encodingType="uint8">
+            <validValue name="Mean">0</validValue>
+            <validValue name="Stdev">1</validValue>
+            <validValue name="MeanMagnitude">2</validValue>
+        </enum>
+    </types>
+
+    <!-- Message 1: Static information -->
+    <sbe:message name="StaticInfo" id="1" description="Static information for hardware, software, and model">
+        <!-- Fixed length fields. These always appear first in SBE -->
+        <field name="time" id="1" type="int64"/>
+        <field name="fieldsPresent" id="2" type="InitFieldsPresent"/>
+        <field name="hwJvmProcessors" id="3" type="uint16"/>
+        <field name="hwNumDevices" id="4" type="uint8"/>
+        <field name="hwJvmMaxMemory" id="5" type="int64"/>
+        <field name="hwOffheapMaxMemory" id="6" type="int64"/>
+        <field name="modelNumLayers" id="7" type="int32"/>
+        <field name="modelNumParams" id="8" type="int64"/>
+        <!-- Groups - always second in SBE. Two groups here: GPU/Device info, and model param names -->
+        <group name="hwDeviceInfoGroup" id="9" dimensionType="groupSizeEncoding">
+            <field name="deviceMemoryMax" id="10" type="int64"/>
+            <data name="deviceDescription" id="50" type="VarDataUTF8"/> <!-- Variable length: last field in a group; must be encoded at end -->
+        </group>
+        <group name="swEnvironmentInfo" id="12" dimensionType="groupSizeEncoding">
+            <data name="envKey" id="51" type="VarDataUTF8"/>
+            <data name="envValue" id="52" type="VarDataUTF8"/>
+        </group>
+        <group name="modelParamNames" id="11" dimensionType="groupSizeEncoding">
+            <data name="modelParamNames" id="53" type="VarDataUTF8"/>
+        </group>
+        <!-- Variable Length Data. By SBE design, all variable length fields must be encoded after fields and groups.
+         Also can't have variable length in composite types, hence separate data fields instead of grouped by sw/hw/model -->
+        <data name="sessionID" id="100" type="VarDataUTF8"/>
+        <data name="typeID" id="101" type="VarDataUTF8"/>
+        <data name="workerID" id="102" type="VarDataUTF8"/>
+        <data name="swArch" id="201" type="VarDataUTF8"/>
+        <data name="swOsName" id="202" type="VarDataUTF8"/>
+        <data name="swJvmName" id="203" type="VarDataUTF8"/>
+        <data name="swJvmVersion" id="204" type="VarDataUTF8"/>
+        <data name="swJvmSpecVersion" id="205" type="VarDataUTF8"/>
+        <data name="swNd4jBackendClass" id="206" type="VarDataUTF8"/>
+        <data name="swNd4jDataTypeName" id="207" type="VarDataUTF8"/>
+        <data name="swHostName" id="208" type="VarDataUTF8"/>
+        <data name="swJvmUID" id="209" type="VarDataUTF8"/>
+        <data name="hwHardwareUID" id="300" type="VarDataUTF8"/>
+        <data name="modelConfigClassName" id="400" type="VarDataUTF8"/>
+        <data name="modelConfigJson" id="401" type="VarDataUTF8"/>
+    </sbe:message>
+
+    <!-- Message 2: Updates -->
+    <sbe:message name="Update" id="2" description="Model status update">
+        <!-- Fixed length fields. These always appear first in SBE -->
+        <field name="time" id="1" type="int64"/>
+        <field name="deltaTime" id="2" type="int32"/>   <!-- Time since last report, MS -->
+        <field name="iterationCount" id="3" type="int32"/>
+        <field name="fieldsPresent" id="4" type="UpdateFieldsPresent"/>
+        <field name="statsCollectionDuration" id="5" type="int32"/>
+        <field name="score" id="6" type="double"/>
+
+        <!-- Groups - always second in SBE -->
+        <group name="memoryUse" id="100" dimensionType="groupSizeEncoding">
+            <field name="memoryType" id="101" type="MemoryType"/>
+            <field name="memoryBytes" id="102" type="int64"/>
+        </group>
+        <group name="performance" id="200" dimensionType="groupSizeEncoding">
+            <field name="totalRuntimeMs" id="201" type="int64"/>
+            <field name="totalExamples" id="202" type="int64"/>
+            <field name="totalMinibatches" id="203" type="int64"/>
+            <field name="examplesPerSecond" id="204" type="float"/>
+            <field name="minibatchesPerSecond" id="205" type="float"/>
+        </group>
+        <group name="gcStats" id="300" dimensionType="groupSizeEncoding">
+            <field name="deltaGCCount" id="301" type="int32"/>
+            <field name="deltaGCTimeMs" id="302" type="int32"/>
+            <data name="gcName" id="1000" type="VarDataUTF8"/>
+        </group>
+        <group name="paramNames" id="350" dimensionType="groupSizeEncoding">
+            <data name="paramName" id="1100" type="VarDataUTF8"/>
+        </group>
+        <group name="layerNames" id="351" dimensionType="groupSizeEncoding">
+            <data name="layerName" id="1101" type="VarDataUTF8"/>
+        </group>
+        <!-- Per parameter and per stats: summary and histograms. mean/stdev/meanMag for each of params/gradients/updates/activations;
+         histograms for params/gradients/updates/activations. Assumption here is that per param stats are encoded first, followed by
+          per layer stats-->
+        <group name="perParameterStats" id="400" dimensionType="groupSizeEncoding">
+            <field name="learningRate" id="401" type="float"/>
+            <group name="summaryStat" id="402" dimensionType="groupSizeEncoding">
+                <field name="statType" id="403" type="StatsType"/>       <!-- Parameters, Gradients, Updates, Activations -->
+                <field name="summaryType" id="404" type="SummaryType"/>   <!-- Mean, stdev, mean magnitude -->
+                <field name="value" id="405" type="double"/>
+            </group>
+            <group name="histograms" id="406" dimensionType="groupSizeEncoding">
+                <field name="statType" id="407" type="StatsType"/>      <!-- Parameters, Gradients, Updates, Activations -->
+                <field name="minValue" id="408" type="double"/>
+                <field name="maxValue" id="409" type="double"/>
+                <field name="nBins" id="410" type="int32"/>
+                <group name="histogramCounts" id="411" dimensionType="groupSizeEncoding">
+                    <field name="binCount" id="412" type="uint32"/>
+                </group>
+            </group>
+        </group>
+        <group name="dataSetMetaDataBytes" id="500" dimensionType="groupSizeEncoding">
+            <group name="metaDataBytes" id="501" dimensionType="groupSizeEncoding">
+                <field name="bytes" id="502" type="int8"/>
+            </group>
+        </group>
+        <!-- Variable Length Data. By SBE design, all variable length fields must be encoded after fields and groups-->
+        <data name="sessionID" id="1200" type="VarDataUTF8"/>
+        <data name="typeID" id="1201" type="VarDataUTF8"/>
+        <data name="workerID" id="1202" type="VarDataUTF8"/>
+        <data name="dataSetMetaDataClassName" id="1300" type="VarDataUTF8"/>
+    </sbe:message>
+
+    <!-- Message 3: Simple MetaData for class for SbeStorageMetaData -->
+    <sbe:message name="StorageMetaData" id="3" description="StorageMetaData">
+        <!-- Fixed length fields: only timestamp -->
+        <field name="timeStamp" id="1" type="int64"/>
+        <!-- Groups: only 1 group - ExtraMetaDataBytes -->
+        <group name="extraMetaDataBytes" id="2" description="Extra metadata bytes">
+            <field name="bytes" id="3" type="int8"/>
+        </group>
+        <!-- Variable length data -->
+        <data name="sessionID" id="4" type="VarDataUTF8"/>
+        <data name="typeID" id="5" type="VarDataUTF8"/>
+        <data name="workerID" id="6" type="VarDataUTF8"/>
+        <data name="initTypeClass" id="7" type="VarDataUTF8"/>
+        <data name="updateTypeClass" id="8" type="VarDataUTF8"/>
+    </sbe:message>
+</sbe:messageSchema>

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/resources/log4j.properties

@@ -0,0 +1,36 @@
+
+#
+# /* ******************************************************************************
+#  *
+#  *
+#  * This program and the accompanying materials are made available under the
+#  * terms of the Apache License, Version 2.0 which is available at
+#  * https://www.apache.org/licenses/LICENSE-2.0.
+#  *
+#  *  See the NOTICE file distributed with this work for additional
+#  *  information regarding copyright ownership.
+#  * Unless required by applicable law or agreed to in writing, software
+#  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  * License for the specific language governing permissions and limitations
+#  * under the License.
+#  *
+#  * SPDX-License-Identifier: Apache-2.0
+#  ******************************************************************************/
+#
+
+log4j.rootLogger=ERROR, Console
+log4j.appender.Console=org.apache.log4j.ConsoleAppender
+log4j.appender.Console.layout=org.apache.log4j.PatternLayout
+log4j.appender.Console.layout.ConversionPattern=%d{ABSOLUTE} %-5p ~ %m%n
+
+log4j.appender.org.springframework=DEBUG
+log4j.appender.org.deeplearning4j=DEBUG
+log4j.appender.org.nd4j=DEBUG
+log4j.appender.org.eclipse.jetty=OFF
+
+log4j.logger.org.springframework=INFO
+log4j.logger.org.deeplearning4j=DEBUG
+log4j.logger.org.nd4j=DEBUG
+log4j.logger.org.eclipse.jetty=OFF
+

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/resources/logback.xml

@@ -0,0 +1,57 @@
+<!--
+  ~ /* ******************************************************************************
+  ~  *
+  ~  *
+  ~  * This program and the accompanying materials are made available under the
+  ~  * terms of the Apache License, Version 2.0 which is available at
+  ~  * https://www.apache.org/licenses/LICENSE-2.0.
+  ~  *
+  ~  *  See the NOTICE file distributed with this work for additional
+  ~  *  information regarding copyright ownership.
+  ~  * Unless required by applicable law or agreed to in writing, software
+  ~  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  ~  * License for the specific language governing permissions and limitations
+  ~  * under the License.
+  ~  *
+  ~  * SPDX-License-Identifier: Apache-2.0
+  ~  ******************************************************************************/
+  -->
+
+<configuration>
+
+
+
+    <appender name="FILE" class="ch.qos.logback.core.FileAppender">
+        <file>logs/application.log</file>
+        <encoder>
+            <pattern> %logger{15} - %message%n%xException{5}
+            </pattern>
+        </encoder>
+    </appender>
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern> %logger{15} - %message%n%xException{5}
+            </pattern>
+        </encoder>
+    </appender>
+
+    <logger name="org.apache.catalina.core" level="DEBUG" />
+    <logger name="org.springframework" level="DEBUG" />
+    <logger name="org.deeplearning4j" level="DEBUG" />
+    <logger name="org.nd4j" level="DEBUG" />
+    <logger name="org.eclipse.jetty" level="INFO" />
+
+    <logger name="opennlp.uima.util" level="OFF" />
+    <logger name="org.apache.uima" level="OFF" />
+    <logger name="org.cleartk" level="OFF" />
+
+
+
+    <root level="ERROR">
+        <appender-ref ref="STDOUT" />
+        <appender-ref ref="FILE" />
+    </root>
+
+</configuration>