chore: import upstream snapshot with attribution

contrib/unified-profiler-analyzer/README.md

@@ -0,0 +1,11 @@
+# Event Logger Analyzer.
+
+This proof of concept tool is for analyzing event logs from the EventLogger and UnifiedProfiler.
+When trying to debug off heap memory leaks it can be hard to understand the allocation patterns.
+
+The UnifiedProfiler emits logs in both an aggregate mode and singular mode which can 
+either output just runtime and aggregate metrics for workspaces or individual allocation
+events for further analysis.
+
+This folder contains classes for helping analyze those logs by converting them to arrow
+for some basic visualization or import in to a proper analytics database for analysis.

contrib/unified-profiler-analyzer/pom.xml

@@ -0,0 +1,69 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.example</groupId>
+  <artifactId>unified-profiler-analyzer</artifactId>
+  <version>1.0-SNAPSHOT</version>
+  <packaging>jar</packaging>
+
+  <name>unified-profiler-analyzer</name>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <maven.compiler.source>11</maven.compiler.source>
+    <maven.compiler.target>11</maven.compiler.target>
+    <dl4j.version>1.0.0-SNAPSHOT</dl4j.version>
+  </properties>
+
+  <dependencies>
+
+    <dependency>
+      <groupId>org.eclipse.deeplearning4j</groupId>
+      <artifactId>datavec-arrow</artifactId>
+      <version>${dl4j.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.deeplearning4j</groupId>
+      <artifactId>datavec-api</artifactId>
+      <version>${dl4j.version}</version>
+    </dependency>
+
+
+    <dependency>
+      <groupId>org.eclipse.deeplearning4j</groupId>
+      <artifactId>datavec-local</artifactId>
+      <version>${dl4j.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-memory-unsafe</artifactId>
+      <version>6.0.1</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.code.findbugs</groupId>
+          <artifactId>jsr305</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <dependency>
+      <groupId>org.eclipse.deeplearning4j</groupId>
+      <artifactId>nd4j-native</artifactId>
+      <version>${dl4j.version}</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.openjfx</groupId>
+        <artifactId>javafx-maven-plugin</artifactId>
+        <version>0.0.8</version>
+        <configuration>
+          <mainClass>org.eclipse.deeplearning4j.profiler.unifiedprofiler.analysis.UnifiedProfilerLogAnalyzer</mainClass>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>

contrib/unified-profiler-analyzer/src/main/java/org/eclipse/deeplearning4j/profiler/unifiedprofiler/analysis/RuntimeSeries.java

@@ -0,0 +1,154 @@
+
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.profiler.unifiedprofiler.analysis;
+
+import javafx.scene.chart.XYChart;
+import javafx.scene.chart.XYChart.Series;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.UInt8Vector;
+import org.nd4j.linalg.profiler.data.eventlogger.LogEvent;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Aggregates metrics for global runtime metrics for rendering.
+ * @author Adam Gibson
+ */
+public class RuntimeSeries {
+    private Series
+            runtimeFreeMemory,
+            runtimeMaxMemory,
+            javacppAvailablePhysicalBytes,
+            javacppMaxBytes,
+            javacppMaxPhysicalBytes,
+            javacppTotalBytes,
+            javacppPointerCount;
+
+
+    public RuntimeSeries() {
+        runtimeFreeMemory = new Series();
+        runtimeMaxMemory = new Series();
+        javacppAvailablePhysicalBytes = new Series();
+        javacppMaxBytes = new Series();
+        javacppMaxPhysicalBytes = new Series();
+        javacppTotalBytes = new Series();
+        javacppPointerCount = new Series();
+    }
+
+
+    /**
+     * Use arrow vectors to record metrics.
+     * @param inputVectors
+     */
+    public synchronized void record(List<FieldVector> inputVectors) {
+        UInt8Vector eventTimeMsVec = (UInt8Vector) inputVectors.get(0);
+        UInt8Vector runtimeMaxMemoryVec = (UInt8Vector) inputVectors.get(6);
+        UInt8Vector runtimeFreeMemoryVec = (UInt8Vector) inputVectors.get(7);
+        UInt8Vector javacppMaxBytesVec = (UInt8Vector) inputVectors.get(8);
+        UInt8Vector javacppMaxPhysicalBytesVec = (UInt8Vector) inputVectors.get(9);
+        UInt8Vector javacppAvailablePhysicalBytesVec = (UInt8Vector) inputVectors.get(10);
+        UInt8Vector javacppTotalBytesVec = (UInt8Vector) inputVectors.get(11);
+        UInt8Vector javacppPointerCountVec = (UInt8Vector) inputVectors.get(12);
+        for(int i  = 0; i < inputVectors.get(0).getValueCount(); i++) {
+            long eventTimeMs = eventTimeMsVec.get(i);
+            long runtimeMaxMemoryVal = runtimeMaxMemoryVec.get(i);
+            long runtimeFreeMemoryVal = runtimeFreeMemoryVec.get(i);
+            long javacppMaxBytesVal = javacppMaxBytesVec.get(i);
+            long javacppMaxPhysicalBytesVal = javacppMaxPhysicalBytesVec.get(i);
+            long javacppAvailablePhysicalBytesVal = javacppAvailablePhysicalBytesVec.get(i);
+            long javacppPointerCountVal = javacppPointerCountVec.get(i);
+            long javacppTotalBytesVal = javacppTotalBytesVec.get(i);
+            runtimeFreeMemory.getData().add(new XYChart.Data<>(runtimeFreeMemoryVal,eventTimeMs));
+            runtimeMaxMemory.getData().add(new XYChart.Data<>(runtimeMaxMemoryVal,eventTimeMs));
+            javacppAvailablePhysicalBytes.getData().add(new XYChart.Data<>(javacppAvailablePhysicalBytesVal,eventTimeMs));
+            javacppMaxBytes.getData().add(new XYChart.Data<>(javacppMaxBytesVal,eventTimeMs));
+            javacppMaxPhysicalBytes.getData().add(new XYChart.Data<>(javacppMaxPhysicalBytesVal,eventTimeMs));
+            javacppTotalBytes.getData().add(new XYChart.Data<>(javacppTotalBytesVal,eventTimeMs));
+            javacppPointerCount.getData().add(new XYChart.Data<>(javacppPointerCountVal,eventTimeMs));
+
+        }
+    }
+
+    public synchronized void record(LogEvent logEvent) {
+        runtimeFreeMemory.getData().add(new XYChart.Data<>(logEvent.getRunTimeMemory().getRuntimeFreeMemory(),logEvent.getEventTimeMs()));
+        runtimeMaxMemory.getData().add(new XYChart.Data<>(logEvent.getRunTimeMemory().getRuntimeMaxMemory(),logEvent.getEventTimeMs()));
+        javacppAvailablePhysicalBytes.getData().add(new XYChart.Data<>(logEvent.getRunTimeMemory().getJavacppAvailablePhysicalBytes(),logEvent.getEventTimeMs()));
+        javacppMaxBytes.getData().add(new XYChart.Data<>(logEvent.getRunTimeMemory().getJavacppMaxBytes(),logEvent.getEventTimeMs()));
+        javacppMaxPhysicalBytes.getData().add(new XYChart.Data<>(logEvent.getRunTimeMemory().getJavacppAvailablePhysicalBytes(),logEvent.getEventTimeMs()));
+        javacppTotalBytes.getData().add(new XYChart.Data<>(logEvent.getRunTimeMemory().getJavacppTotalBytes(),logEvent.getEventTimeMs()));
+        javacppPointerCount.getData().add(new XYChart.Data<>(logEvent.getRunTimeMemory().getJavacppPointerCount(),logEvent.getEventTimeMs()));
+
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        RuntimeSeries that = (RuntimeSeries) o;
+        return Objects.equals(runtimeFreeMemory, that.runtimeFreeMemory) && Objects.equals(runtimeMaxMemory, that.runtimeMaxMemory) && Objects.equals(javacppAvailablePhysicalBytes, that.javacppAvailablePhysicalBytes) && Objects.equals(javacppMaxBytes, that.javacppMaxBytes) && Objects.equals(javacppMaxPhysicalBytes, that.javacppMaxPhysicalBytes);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(runtimeFreeMemory, runtimeMaxMemory, javacppAvailablePhysicalBytes, javacppMaxBytes, javacppMaxPhysicalBytes);
+    }
+
+    public XYChart.Series getRuntimeFreeMemory() {
+        return runtimeFreeMemory;
+    }
+
+    public void setRuntimeFreeMemory(XYChart.Series runtimeFreeMemory) {
+        this.runtimeFreeMemory = runtimeFreeMemory;
+    }
+
+    public XYChart.Series getRuntimeMaxMemory() {
+        return runtimeMaxMemory;
+    }
+
+    public void setRuntimeMaxMemory(XYChart.Series runtimeMaxMemory) {
+        this.runtimeMaxMemory = runtimeMaxMemory;
+    }
+
+    public XYChart.Series getJavacppAvailablePhysicalBytes() {
+        return javacppAvailablePhysicalBytes;
+    }
+
+    public void setJavacppAvailablePhysicalBytes(XYChart.Series javacppAvailablePhysicalBytes) {
+        this.javacppAvailablePhysicalBytes = javacppAvailablePhysicalBytes;
+    }
+
+    public XYChart.Series getJavacppMaxBytes() {
+        return javacppMaxBytes;
+    }
+
+    public void setJavacppMaxBytes(XYChart.Series javacppMaxBytes) {
+        this.javacppMaxBytes = javacppMaxBytes;
+    }
+
+    public XYChart.Series getJavacppMaxPhysicalBytes() {
+        return javacppMaxPhysicalBytes;
+    }
+
+    public void setJavacppMaxPhysicalBytes(XYChart.Series javacppMaxPhysicalBytes) {
+        this.javacppMaxPhysicalBytes = javacppMaxPhysicalBytes;
+    }
+}

contrib/unified-profiler-analyzer/src/main/java/org/eclipse/deeplearning4j/profiler/unifiedprofiler/analysis/UnifiedProfilerAggregateArrowConverter.java

@@ -0,0 +1,139 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.profiler.unifiedprofiler.analysis;
+
+import org.datavec.api.records.mapper.RecordMapper;
+import org.datavec.api.records.reader.impl.csv.CSVRecordReader;
+import org.datavec.api.split.FileSplit;
+import org.datavec.api.split.partition.NumberOfRecordsPartitioner;
+import org.datavec.api.transform.schema.Schema;
+import org.datavec.arrow.recordreader.ArrowRecordWriter;
+
+import java.io.*;
+import java.util.Scanner;
+
+/**
+ * Convert an aggregate based log to an arrow dataset.
+ *
+ * @author Adam Gibson
+ */
+public class UnifiedProfilerAggregateArrowConverter {
+
+    public final static String SPLIT_CSV_DIRECTORY = "split-csv";
+
+    public static void main(String... args) throws Exception {
+        Schema schema = new Schema.Builder()
+                .addColumnLong("eventTimeMs")
+                .addColumnString("associatedWorkspace")
+                .addColumnLong("workspaceSpilledBytes")
+                .addColumnLong("workspacePinnedBytes")
+                .addColumnLong("workspaceExternalBytes")
+                .addColumnLong("workspaceAllocatedMemory")
+                .addColumnLong("runtimeMaxMemory")
+                .addColumnLong("runtimeFreeMemory")
+                .addColumnLong("javacppMaxBytes")
+                .addColumnLong("javacppMaxPhysicalBytes")
+                .addColumnLong("javacppAvailablePhysicalBytes")
+                .addColumnLong("javacppTotalBytes")
+                .addColumnLong("javacppPointerCount")
+                .build();
+
+
+
+        File logFile = new File(args[0]);
+
+        splitLogFile(logFile,10000);
+        CSVRecordReader recordReader = new CSVRecordReader();
+
+        File[] inputFiles = new File(SPLIT_CSV_DIRECTORY).listFiles();
+        for(File inputFile : inputFiles) {
+            FileSplit inputCSv = new FileSplit(inputFile);
+            ArrowRecordWriter arrowRecordWriter = new ArrowRecordWriter(schema);
+            File outputFile = new File("arrow-output/",inputFile.getName() + ".arrow");
+            outputFile.createNewFile();
+            FileSplit outputFileSplit = new FileSplit(outputFile);
+
+            RecordMapper recordMapper = RecordMapper.builder()
+                    .recordReader(recordReader)
+                    .recordWriter(arrowRecordWriter)
+                    .inputUrl(inputCSv)
+                    .batchSize(10000)
+                    .partitioner(new NumberOfRecordsPartitioner())
+                    .outputUrl(outputFileSplit)
+                    .callInitRecordReader(true)
+                    .callInitRecordWriter(true)
+                    .build();
+            recordMapper.copy();
+        }
+
+
+    }
+
+    private static void splitLogFile(File logFile, double numLinesPerFile) throws IOException {
+        Scanner scanner = new Scanner(logFile);
+        int count = 0;
+        while (scanner.hasNextLine()) {
+            scanner.nextLine();
+            count++;
+        }
+        System.out.println("Lines in the file: " + count);     // Displays no. of lines in the input file.
+
+        double temp = (count / numLinesPerFile);
+        int temp1 = (int) temp;
+        int nof = 0;
+        if (temp1 == temp) {
+            nof = temp1;
+        } else {
+            nof = temp1 + 1;
+        }
+        System.out.println("No. of files to be generated :" + nof); // Displays no. of files to be generated.
+
+        //---------------------------------------------------------------------------------------------------------
+
+        // Actual splitting of file into smaller files
+
+        FileInputStream fstream = new FileInputStream(logFile);
+        DataInputStream in = new DataInputStream(fstream);
+
+        BufferedReader br = new BufferedReader(new InputStreamReader(in));
+        String strLine;
+        File inputDirectory = new File(SPLIT_CSV_DIRECTORY);
+        if(!inputDirectory.exists())
+            inputDirectory.mkdirs();
+        for (int j = 1; j <= nof; j++) {
+            FileWriter fstream1 = new FileWriter(SPLIT_CSV_DIRECTORY + File.separator + "event-log-" + j + ".csv");     // Destination File Location
+            BufferedWriter out = new BufferedWriter(fstream1);
+            for (int i = 1; i <= numLinesPerFile; i++) {
+                strLine = br.readLine();
+                if (strLine != null) {
+                    out.write(strLine);
+                    if (i != numLinesPerFile) {
+                        out.newLine();
+                    }
+                }
+            }
+            out.close();
+        }
+
+        in.close();
+
+    }
+
+}

contrib/unified-profiler-analyzer/src/main/java/org/eclipse/deeplearning4j/profiler/unifiedprofiler/analysis/UnifiedProfilerAggregateLogAnalyzer.java

@@ -0,0 +1,190 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.profiler.unifiedprofiler.analysis;
+
+import javafx.application.Application;
+import javafx.embed.swing.SwingFXUtils;
+import javafx.scene.Scene;
+import javafx.scene.chart.LineChart;
+import javafx.scene.chart.NumberAxis;
+import javafx.scene.chart.XYChart;
+import javafx.scene.image.WritableImage;
+import javafx.scene.layout.VBox;
+import javafx.stage.Stage;
+import javafx.util.StringConverter;
+import org.apache.arrow.vector.FieldVector;
+import org.datavec.api.records.reader.impl.transform.TransformProcessRecordReader;
+import org.datavec.api.split.FileSplit;
+import org.datavec.api.transform.TransformProcess;
+import org.datavec.api.transform.schema.Schema;
+import org.datavec.api.writable.Writable;
+import org.datavec.arrow.recordreader.ArrowRecordReader;
+import org.datavec.arrow.recordreader.ArrowWritableRecordBatch;
+import org.nd4j.common.primitives.Counter;
+import org.nd4j.common.primitives.CounterMap;
+import org.nd4j.linalg.api.buffer.DataType;
+import org.nd4j.linalg.profiler.data.eventlogger.EventType;
+import org.nd4j.linalg.profiler.data.eventlogger.ObjectAllocationType;
+
+import javax.imageio.ImageIO;
+import java.io.File;
+import java.io.IOException;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+public class UnifiedProfilerAggregateLogAnalyzer extends Application  {
+
+    private Counter<EventType> eventTypes = new Counter<>();
+    private Counter<ObjectAllocationType> objectAllocationTypes = new Counter<>();
+    private CounterMap<DataType,EventType> eventTypesByDataType = new CounterMap<>();
+    private static File inputFile;
+    private Stage stage;
+    private static ZoneId defaultZoneId = ZoneId.systemDefault();
+    private static //EEE MMM dd HH:mm:ss zzz
+    DateTimeFormatter formatter = DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss zzz",Locale.ENGLISH);
+    public UnifiedProfilerAggregateLogAnalyzer(File inputFile, Stage stage) {
+        this.inputFile = inputFile;
+        this.stage = stage;
+    }
+
+    public UnifiedProfilerAggregateLogAnalyzer() {
+    }
+
+    public void analyze() throws Exception {
+        Map<String,WorkspaceSeries> workspaceMemory = new ConcurrentHashMap<>();
+        RuntimeSeries runtimeSeries = new RuntimeSeries();
+        NumberAxis timeAxis = new NumberAxis();
+        timeAxis.setLabel("Time (in ms)");
+        timeAxis.setTickLabelFormatter(new StringConverter<Number>() {
+            @Override
+            public String toString(Number object) {
+                return new Date(object.longValue()).toString();
+            }
+
+            @Override
+            public Number fromString(String string) {
+                LocalDate date = LocalDate.parse(string, DateTimeFormatter.ISO_DATE);
+                Date date2 = Date.from(date.atStartOfDay(defaultZoneId).toInstant());
+                return date2.getTime();
+
+            }
+        });
+
+        //get the workspace names
+        Schema schema = new Schema.Builder()
+                .addColumnLong("eventTimeMs")
+                .addColumnString("associatedWorkspace")
+                .addColumnLong("workspaceSpilledBytes")
+                .addColumnLong("workspacePinnedBytes")
+                .addColumnLong("workspaceExternalBytes")
+                .addColumnLong("workspaceAllocatedMemory")
+                .addColumnLong("runtimeMaxMemory")
+                .addColumnLong("runtimeFreeMemory")
+                .addColumnLong("javacppMaxBytes")
+                .addColumnLong("javacppMaxPhysicalBytes")
+                .addColumnLong("javacppAvailablePhysicalBytes")
+                .addColumnLong("javacppTotalBytes")
+                .addColumnLong("javacppPointerCount")
+                .build();
+
+
+        ArrowRecordReader arrowRecordReader = new ArrowRecordReader();
+        arrowRecordReader.initialize(new FileSplit(new File("arrow-output")));
+
+        TransformProcess transformProcess = new TransformProcess.Builder(schema)
+                .removeColumns(schema.getColumnNames().stream()
+                        .filter(input -> input.equals("associatedWorkspace"))
+                        .collect(Collectors.toList()))
+                .build();
+
+        TransformProcessRecordReader transformProcessRecordReader = new TransformProcessRecordReader(arrowRecordReader,transformProcess);
+        Set<String> columns = new HashSet<>();
+        while(transformProcessRecordReader.hasNext()) {
+            List<Writable> next = transformProcessRecordReader.next();
+            columns.addAll(next.stream().map(input -> input.toString()).collect(Collectors.toList()));
+        }
+
+        for(String column : columns) {
+            WorkspaceSeries workspaceSeries = new WorkspaceSeries();
+            workspaceMemory.put(column,workspaceSeries);
+        }
+
+
+        arrowRecordReader = new ArrowRecordReader();
+        arrowRecordReader.initialize(new FileSplit(new File("arrow-output")));
+        while(arrowRecordReader.hasNext()) {
+            arrowRecordReader.next();
+            ArrowWritableRecordBatch currentBatch = arrowRecordReader.getCurrentBatch();
+            List<FieldVector> list = currentBatch.getList();
+            runtimeSeries.record(list);
+        }
+
+
+        render(runtimeSeries.getRuntimeMaxMemory(),timeAxis,"Java heap max memory",new File("runtime-max-memory.png"));
+        render(runtimeSeries.getJavacppAvailablePhysicalBytes(),timeAxis,"Javacpp available physical bytes",new File("javacpp-max-physical-bytes.png"));
+        render(runtimeSeries.getJavacppMaxBytes(),timeAxis,"Javacpp maxphysical bytes",new File("javacpp-max-bytes.png"));
+        render(runtimeSeries.getJavacppAvailablePhysicalBytes(),timeAxis,"Javacpp available physical bytes",new File("javacpp-available-physical-bytes.png"));
+        render(runtimeSeries.getRuntimeFreeMemory(),timeAxis,"Java heap free memory",new File("runtime-free-memory.png"));
+
+    }
+
+
+
+
+    private void render(XYChart.Series<Number,Number> series,NumberAxis timeChart,String label,File outputFile) throws IOException {
+        NumberAxis allocatedChart = new NumberAxis();
+        allocatedChart.setLabel(label);
+        LineChart<Number,Number> allocatedChart2 = new LineChart<>(allocatedChart,timeChart);
+        allocatedChart2.getData().add(series);
+        saveImage(allocatedChart2,outputFile);
+
+    }
+
+    private void saveImage(LineChart lineChart,File file) throws IOException {
+        //https://stackoverflow.com/questions/29721289/how-to-generate-chart-image-using-javafx-chart-api-for-export-without-displying
+        //save image as above
+        VBox vbox = new VBox(lineChart);
+
+        Scene scene = new Scene(vbox, 400, 200);
+
+        stage.setScene(scene);
+        stage.setHeight(300);
+        stage.setWidth(1200);
+        WritableImage image = scene.snapshot(null);
+        ImageIO.write(SwingFXUtils.fromFXImage(image, null),
+                "png", file);
+
+    }
+
+    public static void main(String...args) throws Exception {
+        UnifiedProfilerAggregateLogAnalyzer.inputFile = new File(args[0]);
+        Application.launch(args);
+    }
+
+    @Override
+    public void start(Stage stage) throws Exception {
+        UnifiedProfilerAggregateLogAnalyzer unifiedProfilerLogAnalyzer = new UnifiedProfilerAggregateLogAnalyzer(inputFile,stage);
+        unifiedProfilerLogAnalyzer.analyze();
+    }
+}

contrib/unified-profiler-analyzer/src/main/java/org/eclipse/deeplearning4j/profiler/unifiedprofiler/analysis/UnifiedProfilerArrowConverter.java

@@ -0,0 +1,151 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.profiler.unifiedprofiler.analysis;
+
+import org.datavec.api.records.mapper.RecordMapper;
+import org.datavec.api.records.reader.impl.csv.CSVRecordReader;
+import org.datavec.api.split.FileSplit;
+import org.datavec.api.split.partition.NumberOfRecordsPartitioner;
+import org.datavec.api.transform.schema.Schema;
+import org.datavec.arrow.recordreader.ArrowRecordWriter;
+import org.nd4j.linalg.api.buffer.DataType;
+
+import java.io.*;
+import java.util.Arrays;
+import java.util.Scanner;
+import java.util.stream.Collectors;
+
+/**
+ * Converts csv logs from {@link org.nd4j.linalg.profiler.data.eventlogger.EventLogger}
+ * when {@link org.nd4j.linalg.profiler.data.eventlogger.EventLogger#aggregateMode}
+ * is false to arrow format.
+ *
+ * @author Adam Gibson
+ */
+public class UnifiedProfilerArrowConverter {
+
+    public final static String SPLIT_CSV_DIRECTORY = "split-csv";
+
+    public static void main(String... args) throws Exception {
+        Schema schema = new Schema.Builder()
+                .addColumnLong("eventTimeMs")
+                .addColumnCategorical("eventType", "ALLOCATION", "DEALLOCATION")
+                .addColumnCategorical("objectAllocationType", "OP_CONTEXT", "DATA_BUFFER", "WORKSPACE")
+                .addColumnString("associatedWorkspace")
+                .addColumnString("associatedThreadName")
+                .addColumnCategorical("datatype", Arrays.stream(DataType.values()).map(input -> input.name()).collect(Collectors.toList()).toArray(new String[0]))
+                .addColumnLong("memInBytes")
+                .addColumnBoolean("isAttached")
+                .addColumnBoolean("isConstant")
+                .addColumnLong("objectId")
+                .addColumnLong("workspaceAllocatedMemory")
+                .addColumnLong("workspaceExternalBytes")
+                .addColumnLong("workspacePinnedBytes")
+                .addColumnLong("workspaceSpilledBytes")
+                .addColumnLong("runtimeFreeMemory")
+                .addColumnLong("javacppAvailablePhysicalBytes")
+                .addColumnLong("javacppMaxPhysicalBytes")
+                .addColumnLong("javacppMaxBytes")
+                .addColumnLong("javacppPointerCount")
+                .addColumnLong("javacppTotalBytes")
+                .addColumnLong("runtimeMaxMemory")
+                .build();
+
+
+        File logFile = new File("/home/agibsonccc/Documents/GitHub/servicenow-reproducer/event-log.csv");
+
+        splitLogFile(logFile,10000);
+        CSVRecordReader recordReader = new CSVRecordReader();
+
+        File[] inputFiles = new File(SPLIT_CSV_DIRECTORY).listFiles();
+        for(File inputFile : inputFiles) {
+            FileSplit inputCSv = new FileSplit(inputFile);
+            ArrowRecordWriter arrowRecordWriter = new ArrowRecordWriter(schema);
+            File outputFile = new File("arrow-output/",inputFile.getName() + ".arrow");
+            outputFile.createNewFile();
+            FileSplit outputFileSplit = new FileSplit(outputFile);
+
+            RecordMapper recordMapper = RecordMapper.builder()
+                    .recordReader(recordReader)
+                    .recordWriter(arrowRecordWriter)
+                    .inputUrl(inputCSv)
+                    .batchSize(10000)
+                    .partitioner(new NumberOfRecordsPartitioner())
+                    .outputUrl(outputFileSplit)
+                    .callInitRecordReader(true)
+                    .callInitRecordWriter(true)
+                    .build();
+            recordMapper.copy();
+        }
+
+
+    }
+
+    private static void splitLogFile(File logFile, double numLinesPerFile) throws IOException {
+        Scanner scanner = new Scanner(logFile);
+        int count = 0;
+        while (scanner.hasNextLine()) {
+            scanner.nextLine();
+            count++;
+        }
+        System.out.println("Lines in the file: " + count);     // Displays no. of lines in the input file.
+
+        double temp = (count / numLinesPerFile);
+        int temp1 = (int) temp;
+        int nof = 0;
+        if (temp1 == temp) {
+            nof = temp1;
+        } else {
+            nof = temp1 + 1;
+        }
+        System.out.println("No. of files to be generated :" + nof); // Displays no. of files to be generated.
+
+        //---------------------------------------------------------------------------------------------------------
+
+        // Actual splitting of file into smaller files
+
+        FileInputStream fstream = new FileInputStream(logFile);
+        DataInputStream in = new DataInputStream(fstream);
+
+        BufferedReader br = new BufferedReader(new InputStreamReader(in));
+        String strLine;
+        File inputDirectory = new File(SPLIT_CSV_DIRECTORY);
+        if(!inputDirectory.exists())
+            inputDirectory.mkdirs();
+        for (int j = 1; j <= nof; j++) {
+            FileWriter fstream1 = new FileWriter(SPLIT_CSV_DIRECTORY + File.separator + "event-log-" + j + ".csv");     // Destination File Location
+            BufferedWriter out = new BufferedWriter(fstream1);
+            for (int i = 1; i <= numLinesPerFile; i++) {
+                strLine = br.readLine();
+                if (strLine != null) {
+                    out.write(strLine);
+                    if (i != numLinesPerFile) {
+                        out.newLine();
+                    }
+                }
+            }
+            out.close();
+        }
+
+        in.close();
+
+    }
+
+}

contrib/unified-profiler-analyzer/src/main/java/org/eclipse/deeplearning4j/profiler/unifiedprofiler/analysis/UnifiedProfilerLogAnalyzer.java

@@ -0,0 +1,135 @@
+    /*
+     *  ******************************************************************************
+     *  *
+     *  *
+     *  * This program and the accompanying materials are made available under the
+     *  * terms of the Apache License, Version 2.0 which is available at
+     *  * https://www.apache.org/licenses/LICENSE-2.0.
+     *  *
+     *  *  See the NOTICE file distributed with this work for additional
+     *  *  information regarding copyright ownership.
+     *  * Unless required by applicable law or agreed to in writing, software
+     *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+     *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+     *  * License for the specific language governing permissions and limitations
+     *  * under the License.
+     *  *
+     *  * SPDX-License-Identifier: Apache-2.0
+     *  *****************************************************************************
+     */
+    package org.eclipse.deeplearning4j.profiler.unifiedprofiler.analysis;
+
+    import javafx.application.Application;
+    import javafx.embed.swing.SwingFXUtils;
+    import javafx.scene.Scene;
+    import javafx.scene.chart.LineChart;
+    import javafx.scene.chart.NumberAxis;
+    import javafx.scene.image.WritableImage;
+    import javafx.scene.layout.VBox;
+    import javafx.stage.Stage;
+    import org.datavec.api.split.FileSplit;
+    import org.datavec.api.transform.analysis.DataAnalysis;
+    import org.datavec.api.transform.schema.Schema;
+    import org.datavec.api.transform.ui.HtmlAnalysis;
+    import org.datavec.arrow.recordreader.ArrowRecordReader;
+    import org.datavec.local.transforms.AnalyzeLocal;
+    import org.nd4j.common.primitives.Counter;
+    import org.nd4j.common.primitives.CounterMap;
+    import org.nd4j.linalg.api.buffer.DataType;
+    import org.nd4j.linalg.profiler.data.eventlogger.EventType;
+    import org.nd4j.linalg.profiler.data.eventlogger.ObjectAllocationType;
+
+    import javax.imageio.ImageIO;
+    import java.io.File;
+    import java.io.IOException;
+    import java.util.Arrays;
+    import java.util.Map;
+    import java.util.concurrent.ConcurrentHashMap;
+    import java.util.stream.Collectors;
+
+    /**
+     *
+     */
+    public class UnifiedProfilerLogAnalyzer extends Application  {
+
+        private Counter<EventType> eventTypes = new Counter<>();
+        private Counter<ObjectAllocationType> objectAllocationTypes = new Counter<>();
+        private CounterMap<DataType,EventType> eventTypesByDataType = new CounterMap<>();
+        private static File inputFile;
+        private Stage stage;
+
+        public UnifiedProfilerLogAnalyzer(File inputFile,Stage stage) {
+            this.inputFile = inputFile;
+            this.stage = stage;
+        }
+
+        public UnifiedProfilerLogAnalyzer() {
+        }
+
+        public void analyze() throws Exception {
+            Map<String,WorkspaceSeries> workspaceMemory = new ConcurrentHashMap<>();
+            RuntimeSeries runtimeSeries = new RuntimeSeries();
+            NumberAxis timeAxis = new NumberAxis();
+
+            //get the workspace names
+            Schema schema = new Schema.Builder()
+                    .addColumnLong("eventTimeMs")
+                    .addColumnCategorical("eventType","ALLOCATION","DEALLOCATION")
+                    .addColumnCategorical("objectAllocationType","OP_CONTEXT","DATA_BUFFER","WORKSPACE")
+                    .addColumnString("associatedWorkspace")
+                    .addColumnString("associatedThreadName")
+                    .addColumnCategorical("datatype", Arrays.stream(DataType.values()).map(input -> input.name()).collect(Collectors.toList()).toArray(new String[0]))
+                    .addColumnLong("memInBytes")
+                    .addColumnBoolean("isAttached")
+                    .addColumnBoolean("isConstant")
+                    .addColumnLong("objectId")
+                    .addColumnLong("workspaceAllocatedMemory")
+                    .addColumnLong("workspaceExternalBytes")
+                    .addColumnLong("workspacePinnedBytes")
+                    .addColumnLong("workspaceSpilledBytes")
+                    .addColumnLong("runtimeFreeMemory")
+                    .addColumnLong("javacppAvailablePhysicalBytes")
+                    .addColumnLong("javacppMaxPhysicalBytes")
+                    .addColumnLong("javacppMaxBytes")
+                    .addColumnLong("runtimeMaxMemory")
+                    .build();
+
+            ArrowRecordReader arrowRecordReader = new ArrowRecordReader();
+            arrowRecordReader.initialize(new FileSplit(new File("arrow-output")));
+            DataAnalysis analyze = AnalyzeLocal.analyze(schema, arrowRecordReader);
+            HtmlAnalysis.createHtmlAnalysisFile(analyze,new File("analysis.html"));
+
+
+        }
+
+
+
+
+
+        private void saveImage(LineChart lineChart,File file) throws IOException {
+            //https://stackoverflow.com/questions/29721289/how-to-generate-chart-image-using-javafx-chart-api-for-export-without-displying
+            //save image as above
+            VBox vbox = new VBox(lineChart);
+
+            Scene scene = new Scene(vbox, 400, 200);
+
+            stage.setScene(scene);
+            stage.setHeight(300);
+            stage.setWidth(1200);
+            WritableImage image = scene.snapshot(null);
+            ImageIO.write(SwingFXUtils.fromFXImage(image, null),
+                    "png", file);
+
+        }
+
+        public static void main(String...args) throws Exception {
+            UnifiedProfilerLogAnalyzer.inputFile = new File(args[0]);
+            Application.launch(args);
+        }
+
+        @Override
+        public void start(Stage stage) throws Exception {
+            UnifiedProfilerLogAnalyzer unifiedProfilerLogAnalyzer = new UnifiedProfilerLogAnalyzer(inputFile,stage);
+            unifiedProfilerLogAnalyzer.analyze();
+        }
+    }

contrib/unified-profiler-analyzer/src/main/java/org/eclipse/deeplearning4j/profiler/unifiedprofiler/analysis/WorkspaceSeries.java

@@ -0,0 +1,108 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.eclipse.deeplearning4j.profiler.unifiedprofiler.analysis;
+
+import javafx.scene.chart.XYChart;
+import org.nd4j.linalg.profiler.data.eventlogger.LogEvent;
+
+import java.util.Objects;
+
+/**
+ * Aggregates workspace metrics
+ * from {@link org.nd4j.linalg.profiler.data.eventlogger.EventLogger}
+ * when {@link org.nd4j.linalg.profiler.data.eventlogger.EventLogger#aggregateMode}
+ * is false.
+ *
+ * @author Adam Gibson
+ */
+
+public class WorkspaceSeries {
+    private XYChart.Series spilled,external,allocated,pinned;
+
+    private long eventTimeMs;
+    public WorkspaceSeries() {
+        spilled = new XYChart.Series();
+        external = new XYChart.Series();
+        allocated = new XYChart.Series();
+        pinned = new XYChart.Series();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        WorkspaceSeries that = (WorkspaceSeries) o;
+        return eventTimeMs == that.eventTimeMs && Objects.equals(spilled, that.spilled) && Objects.equals(external, that.external) && Objects.equals(allocated, that.allocated) && Objects.equals(pinned, that.pinned);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(spilled, external, allocated, pinned, eventTimeMs);
+    }
+
+    public long getEventTimeMs() {
+        return eventTimeMs;
+    }
+
+    public void setEventTimeMs(long eventTimeMs) {
+        this.eventTimeMs = eventTimeMs;
+    }
+
+    public XYChart.Series getSpilled() {
+        return spilled;
+    }
+
+    public void setSpilled(XYChart.Series spilled) {
+        this.spilled = spilled;
+    }
+
+    public XYChart.Series getExternal() {
+        return external;
+    }
+
+    public void setExternal(XYChart.Series external) {
+        this.external = external;
+    }
+
+    public XYChart.Series getAllocated() {
+        return allocated;
+    }
+
+    public void setAllocated(XYChart.Series allocated) {
+        this.allocated = allocated;
+    }
+
+    public XYChart.Series getPinned() {
+        return pinned;
+    }
+
+    public void setPinned(XYChart.Series pinned) {
+        this.pinned = pinned;
+    }
+
+    public synchronized void record(LogEvent logEvent) {
+        spilled.getData().add(new XYChart.Data(logEvent.getEventTimeMs(),logEvent.getWorkspaceInfo().getSpilledBytes(),logEvent.getEventTimeMs()));
+        external.getData().add(new XYChart.Data(logEvent.getEventTimeMs(),logEvent.getWorkspaceInfo().getExternalBytes(),logEvent.getEventTimeMs()));
+        allocated.getData().add(new XYChart.Data(logEvent.getEventTimeMs(),logEvent.getWorkspaceInfo().getAllocatedMemory(),logEvent.getEventTimeMs()));
+        pinned.getData().add(new XYChart.Data(logEvent.getEventTimeMs(),logEvent.getWorkspaceInfo().getPinnedBytes(),logEvent.getEventTimeMs()));
+    }
+    
+
+}