chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,9 @@
|
||||
Contrib folder
|
||||
---------------------------
|
||||
|
||||
This folder contains supplementary and retired code for the Eclipse Deeplearning4j project.
|
||||
|
||||
1. Attic: These are modules that are no longer maintained, but kept in this repository for posterity.
|
||||
|
||||
2. Codegen tools: Supplementary code for generating op definitions, and unit testing utilities for deeplearning4j
|
||||
proper.
|
||||
@@ -0,0 +1,16 @@
|
||||
*.class
|
||||
|
||||
# Mobile Tools for Java (J2ME)
|
||||
.mtj.tmp/
|
||||
|
||||
# Package Files #
|
||||
*.jar
|
||||
*.war
|
||||
*.ear
|
||||
|
||||
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
|
||||
hs_err_pid*
|
||||
target/
|
||||
|
||||
*.iml
|
||||
.idea/
|
||||
@@ -0,0 +1,38 @@
|
||||
# benchmarking_nd4j
|
||||
Simple Microbenchmarks for ND4J utilizing the JMH
|
||||
|
||||
Original credit to https://github.com/treo/benchmarking_nd4j
|
||||
|
||||
## Building
|
||||
|
||||
mvn clean package
|
||||
|
||||
## Running
|
||||
|
||||
java -jar target/benchmarks.jar
|
||||
|
||||
|
||||
For a quick run, e.g. just the same Matrix Size progression as used in the Neanderthal benchmarks, run like this:
|
||||
|
||||
java -jar target/benchmarks.jar -f2 -i10 -wi 2 Neanderthal
|
||||
|
||||
## Running the memory profiler to detect memory leaks
|
||||
Ensure jemalloc is installed with the profiler enabled. Something like:
|
||||
wget https://github.com/jemalloc/jemalloc/releases/download/5.2.0/jemalloc-5.2.0.tar.bz2 && \
|
||||
tar -xvf jemalloc-5.2.0.tar.bz2 && \
|
||||
cd jemalloc-5.2.0 && \
|
||||
./configure --enable-prof && \
|
||||
make && \
|
||||
make install
|
||||
|
||||
Run memory-prof.sh
|
||||
|
||||
This will run the above build process and the [MemoryPressureTest](src/main/java/org/nd4j/benchmark/memory/MemoryPressureTest.java) which will allocate a large amount of memory and then free it. The jemalloc profiler will be used to detect memory leaks.
|
||||
|
||||
## Choosing a BLAS Library
|
||||
|
||||
Since ND4J supports multiple blas libraries, you have to specify which one you actually want to use. For my own benchmarks I've been using MKL.
|
||||
|
||||
-Dorg.bytedeco.javacpp.openblas.load=mkl_rt
|
||||
|
||||
For more information see https://github.com/bytedeco/javacpp-presets/tree/master/openblas
|
||||
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
LIBJEMALLOC="/home/agibsonccc/jemalloc-5.3.0/lib/libjemalloc.so.2"
|
||||
MALLOC_CONF="prof:true,lg_prof_interval:31,lg_prof_sample:17,prof_prefix:jeprof.out"
|
||||
mvn package
|
||||
|
||||
LD_PRELOAD="$LIBJEMALLOC" \
|
||||
MALLOC_CONF="$MALLOC_CONF" \
|
||||
java \
|
||||
-cp target/benchmarks.jar \
|
||||
org.nd4j.memorypressure.MemoryPressureTest \
|
||||
-f2 \
|
||||
-i10 \
|
||||
-wi 2 \
|
||||
|
||||
jeprof --show_bytes --gif $(which java) jeprof.*.heap > app-profiling.gif
|
||||
@@ -0,0 +1,136 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>nd4j-benchmark</artifactId>
|
||||
<version>0.1.0-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>ND4J Benchmarks</name>
|
||||
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<jmh.version>1.37</jmh.version>
|
||||
<javac.target>11</javac.target>
|
||||
<java.version>11</java.version>
|
||||
<java.compiler.version>11</java.compiler.version>
|
||||
|
||||
<uberjar.name>benchmarks</uberjar.name>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<nd4j.version>1.0.0-SNAPSHOT</nd4j.version>
|
||||
</properties>
|
||||
|
||||
|
||||
<!--
|
||||
This is the demo/sample template build script for building Java benchmarks with JMH.
|
||||
Edit as needed.
|
||||
-->
|
||||
|
||||
<prerequisites>
|
||||
<maven>3.0</maven>
|
||||
</prerequisites>
|
||||
|
||||
|
||||
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.openjdk.jmh</groupId>
|
||||
<artifactId>jmh-generator-annprocess</artifactId>
|
||||
<version>${jmh.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.openjdk.jmh</groupId>
|
||||
<artifactId>jmh-core</artifactId>
|
||||
<version>${jmh.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>nd4j-native</artifactId>
|
||||
<version>${nd4j.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.bytedeco</groupId>
|
||||
<artifactId>openblas-platform</artifactId>
|
||||
<version>0.3.26-1.5.10</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>jul-to-slf4j</artifactId>
|
||||
<version>1.7.36</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
<version>1.5.12</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.8.1</version>
|
||||
<configuration>
|
||||
<source>${java.version}</source>
|
||||
<target>${javac.target}</target>
|
||||
<compilerVersion>${java.compiler.version}</compilerVersion>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>3.5.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<finalName>${uberjar.name}</finalName>
|
||||
<transformers>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
|
||||
<mainClass>org.openjdk.jmh.Main</mainClass>
|
||||
</transformer>
|
||||
</transformers>
|
||||
<filters>
|
||||
<filter>
|
||||
<artifact>uncomplicate:neanderthal</artifact>
|
||||
<includes>
|
||||
<include>**</include>
|
||||
</includes>
|
||||
</filter>
|
||||
<filter>
|
||||
<artifact>org.clojure:clojure</artifact>
|
||||
<excludes>
|
||||
<exclude>clojure/pprint/proxy$java/**</exclude>
|
||||
</excludes>
|
||||
</filter>
|
||||
<filter>
|
||||
<!--
|
||||
Shading signed JARs will fail without this.
|
||||
http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar
|
||||
-->
|
||||
<artifact>*:*</artifact>
|
||||
<excludes>
|
||||
<exclude>META-INF/*.SF</exclude>
|
||||
<exclude>META-INF/*.DSA</exclude>
|
||||
<exclude>META-INF/*.RSA</exclude>
|
||||
</excludes>
|
||||
</filter>
|
||||
</filters>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,88 @@
|
||||
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j;
|
||||
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.factory.Nd4j;
|
||||
import org.openjdk.jmh.annotations.*;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class BlasWrapper {
|
||||
|
||||
@State(Scope.Thread)
|
||||
public static class SetupState {
|
||||
public INDArray array1 = Nd4j.ones(100).addi(0.01f);
|
||||
public INDArray array2 = Nd4j.ones(100).addi(0.01f);
|
||||
public INDArray array3 = Nd4j.ones(100).addi(0.01f);
|
||||
|
||||
|
||||
public org.nd4j.linalg.factory.BlasWrapper wrapper = Nd4j.getBlasWrapper();
|
||||
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void asum(SetupState state) {
|
||||
state.wrapper.asum(state.array1);
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void axpy(SetupState state) {
|
||||
state.wrapper.axpy(new Float(0.75f), state.array1, state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void copy(SetupState state) {
|
||||
state.wrapper.copy(state.array1, state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void dot(SetupState state) {
|
||||
state.wrapper.dot(state.array1, state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void nrm2(SetupState state) {
|
||||
state.wrapper.nrm2(state.array1);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void iamax(SetupState state) {
|
||||
state.wrapper.iamax(state.array1);
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void swap(SetupState state) {
|
||||
state.wrapper.swap(state.array1, state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void gemv(SetupState state) {
|
||||
state.wrapper.gemv((Number) new Float(0.75f), state.array1, state.array2, new Double(0.5), state.array3);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void ger(SetupState state) {
|
||||
state.wrapper.ger(new Double(0.75f), state.array1, state.array2, state.array3);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j;
|
||||
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.openjdk.jmh.annotations.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class Flattening {
|
||||
|
||||
@State(Scope.Thread)
|
||||
public static class SetupState {
|
||||
public INDArray small_c = org.nd4j.linalg.factory.Nd4j.create(new int[]{1<<10, 1<<10}, 'c');
|
||||
public INDArray small_f = org.nd4j.linalg.factory.Nd4j.create(new int[]{1<<10, 1<<10}, 'f');
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void toFlattened_CC_Small(SetupState state) throws IOException {
|
||||
org.nd4j.linalg.factory.Nd4j.toFlattened('c', state.small_c);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void toFlattened_CF_Small(SetupState state) throws IOException {
|
||||
org.nd4j.linalg.factory.Nd4j.toFlattened('f', state.small_c);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void toFlattened_FF_Small(SetupState state) throws IOException {
|
||||
org.nd4j.linalg.factory.Nd4j.toFlattened('f', state.small_f);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void toFlattened_FC_Small(SetupState state) throws IOException {
|
||||
org.nd4j.linalg.factory.Nd4j.toFlattened('c', state.small_f);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j;
|
||||
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.factory.Nd4j;
|
||||
import org.openjdk.jmh.annotations.*;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class Large_NDArray {
|
||||
|
||||
@State(Scope.Thread)
|
||||
public static class SetupState {
|
||||
public INDArray array1 = Nd4j.ones(1<<28);
|
||||
public INDArray array2 = Nd4j.ones(1<<28);
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS)
|
||||
public void sumNumber(SetupState state) {
|
||||
state.array1.sumNumber().doubleValue();
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS)
|
||||
public void add(SetupState state) {
|
||||
state.array1.add(state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS)
|
||||
public void addi(SetupState state) {
|
||||
state.array1.addi(state.array2);
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS)
|
||||
public void sub(SetupState state) {
|
||||
state.array1.sub(state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS)
|
||||
public void subi(SetupState state) {
|
||||
state.array1.subi(state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS)
|
||||
public void mul(SetupState state) {
|
||||
state.array1.mul(state.array2);
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS)
|
||||
public void muli(SetupState state) {
|
||||
state.array1.muli(state.array2);
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS)
|
||||
public void assign(SetupState state) {
|
||||
state.array1.assign(state.array2);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j;
|
||||
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.factory.Nd4j;
|
||||
import org.openjdk.jmh.annotations.*;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class Medium_NDArray {
|
||||
|
||||
|
||||
@State(Scope.Thread)
|
||||
public static class SetupState {
|
||||
public INDArray array1 = Nd4j.ones(1<<16);
|
||||
public INDArray array2 = Nd4j.ones(1<<16);
|
||||
|
||||
static {
|
||||
// Only needed for mkl on RC3.8
|
||||
//System.loadLibrary("mkl_rt");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void sumNumber(SetupState state) {
|
||||
state.array1.sumNumber().doubleValue();
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void add(SetupState state) {
|
||||
state.array1.add(state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void addi(SetupState state) {
|
||||
state.array1.addi(state.array2);
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void sub(SetupState state) {
|
||||
state.array1.sub(state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void subi(SetupState state) {
|
||||
state.array1.subi(state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void mul(SetupState state) {
|
||||
state.array1.mul(state.array2);
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void muli(SetupState state) {
|
||||
state.array1.muli(state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void cumsum(SetupState state) {
|
||||
state.array1.cumsum(0);
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void cumsumi(SetupState state) {
|
||||
state.array1.cumsumi(0);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void assign(SetupState state) {
|
||||
state.array1.assign(state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void mmul(SetupState state) {
|
||||
state.array1.mmul(state.array2);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j;
|
||||
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.api.ops.impl.scalar.ScalarMultiplication;
|
||||
import org.nd4j.linalg.api.ops.impl.transforms.strict.Exp;
|
||||
import org.nd4j.linalg.factory.Nd4j;
|
||||
import org.openjdk.jmh.annotations.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class NativeOps {
|
||||
|
||||
@State(Scope.Thread)
|
||||
public static class SetupState {
|
||||
INDArray array = Nd4j.ones(1024, 1024);
|
||||
INDArray arrayRow = Nd4j.linspace(1, 1024, 1024);
|
||||
INDArray arrayColumn = Nd4j.linspace(1, 1024, 1024).reshape(1024,1);
|
||||
INDArray array1 = Nd4j.linspace(1, 20480, 20480);
|
||||
INDArray array2 = Nd4j.linspace(1, 20480, 20480);
|
||||
|
||||
INDArray array3 = Nd4j.ones(128, 256);
|
||||
INDArray arrayRow3 = Nd4j.linspace(1, 256, 256);
|
||||
|
||||
INDArray arrayUnordered = Nd4j.ones(512, 512);
|
||||
INDArray arrayOrderedC = Nd4j.zeros(512, 512,'c');
|
||||
INDArray arrayOrderedF = Nd4j.zeros(512, 512, 'f');
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void broadcastColumn(SetupState state) throws IOException {
|
||||
state.array.addiColumnVector(state.arrayColumn);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void broadcastRow(SetupState state) throws IOException {
|
||||
state.array.addiRowVector(state.arrayRow);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void transformOp(SetupState state) throws IOException {
|
||||
Nd4j.getExecutioner().exec(new Exp(state.array1, state.array2));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void scalarOp2(SetupState state) throws IOException {
|
||||
Nd4j.getExecutioner().exec(new ScalarMultiplication(state.arrayUnordered, 2.5f));
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void dupDifferentOrdersOp(SetupState state) throws IOException {
|
||||
state.arrayUnordered.assign(state.arrayOrderedF);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void dupSameOrdersOp(SetupState state) throws IOException {
|
||||
state.arrayUnordered.assign(state.arrayOrderedC);
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void pairwiseOp1(SetupState state) throws IOException {
|
||||
state.array1.addiRowVector(state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void broadcastOp2(SetupState state) throws IOException {
|
||||
state.array.addiRowVector(state.arrayRow);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void reduceOp1(SetupState state) throws IOException {
|
||||
state.array.sum(0);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void reduceOp2(SetupState state) throws IOException {
|
||||
state.array.sumNumber().floatValue();
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS)
|
||||
public void scalarOp1(SetupState state) throws IOException {
|
||||
state.array2.addi(0.5f);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j;
|
||||
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.factory.Nd4j;
|
||||
import org.nd4j.nativeblas.NativeOpsHolder;
|
||||
import org.openjdk.jmh.annotations.*;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class Small_NDArray {
|
||||
|
||||
|
||||
@State(Scope.Thread)
|
||||
public static class SetupState {
|
||||
public INDArray array1 = Nd4j.ones(1<<8);
|
||||
public INDArray array2 = Nd4j.ones(1<<8);
|
||||
|
||||
@Setup
|
||||
public void setup(){
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void sumNumber(SetupState state) {
|
||||
state.array1.sumNumber().doubleValue();
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void add(SetupState state) {
|
||||
state.array1.add(state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void addi(SetupState state) {
|
||||
state.array1.addi(state.array2);
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void sub(SetupState state) {
|
||||
state.array1.sub(state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void subi(SetupState state) {
|
||||
state.array1.subi(state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void mul(SetupState state) {
|
||||
state.array1.mul(state.array2);
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void muli(SetupState state) {
|
||||
state.array1.muli(state.array2);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void cumsum(SetupState state) {
|
||||
state.array1.cumsum(0);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void cumsumi(SetupState state) {
|
||||
state.array1.cumsumi(0);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void assign(SetupState state) {
|
||||
state.array1.assign(state.array2);
|
||||
}
|
||||
|
||||
}
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.bypass;
|
||||
|
||||
import org.bytedeco.javacpp.FloatPointer;
|
||||
import org.bytedeco.openblas.global.openblas;
|
||||
import org.nd4j.linalg.api.blas.Level3;
|
||||
import org.nd4j.linalg.api.blas.params.GemmParams;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.factory.Nd4j;
|
||||
import org.openjdk.jmh.annotations.*;
|
||||
import org.openjdk.jmh.infra.Blackhole;
|
||||
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class BypassComparison_128x128 {
|
||||
|
||||
|
||||
@State(Scope.Thread)
|
||||
public static class SetupState {
|
||||
public int size = 128;
|
||||
public INDArray m1 = Nd4j.ones(size, size);
|
||||
public INDArray m2 = Nd4j.ones(m1.shape());
|
||||
public INDArray r = Nd4j.createUninitialized(m1.shape(), 'f');
|
||||
|
||||
|
||||
public GemmParams params = new GemmParams(m1, m2, r);
|
||||
FloatPointer a = (FloatPointer) params.getA().data().addressPointer();
|
||||
FloatPointer b = (FloatPointer) params.getB().data().addressPointer();
|
||||
FloatPointer c = (FloatPointer) params.getC().data().addressPointer();
|
||||
|
||||
int M = params.getM();
|
||||
int N = params.getN();
|
||||
int K = params.getK();
|
||||
int lda = params.getLda();
|
||||
int ldb = params.getLdb();
|
||||
int ldc = params.getLdc();
|
||||
|
||||
|
||||
public Level3 wrapper = Nd4j.getBlasWrapper().level3();
|
||||
public Method sgemm;
|
||||
|
||||
@Setup(Level.Iteration)
|
||||
public void doSetup(){
|
||||
try {
|
||||
sgemm = wrapper.getClass().getDeclaredMethod("sgemm", char.class, char.class, char.class, int.class, int.class, int.class, float.class, INDArray.class,
|
||||
int.class, INDArray.class, int.class, float.class, INDArray.class, int.class);
|
||||
sgemm.setAccessible(true);
|
||||
} catch (NoSuchMethodException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void mmuli(SetupState state) {
|
||||
state.m1.mmuli(state.m2, state.r);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void nd4j_gemm(SetupState state) {
|
||||
Nd4j.gemm(state.m1, state.m2, state.r, false, false, 1.0, 0.0);
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void openblas_cblas_gemm(SetupState state, Blackhole bh) {
|
||||
openblas.cblas_sgemm(102,111, 111, state.M, state.N, state.K, 1.0f, state.a, state.lda, state.b, state.ldb, 0.0f, state.c, state.ldc);
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void level3_sgemm(SetupState state, Blackhole bh) {
|
||||
final GemmParams params = state.params;
|
||||
try {
|
||||
state.sgemm.invoke(state.wrapper, params.getA().ordering(), params.getTransA(), params.getTransB(), params.getM(), params.getN(), params.getK(), (float)1.0, params.getA(), params.getLda(), params.getB(), params.getLdb(), (float)0.0, params.getC(), params.getLdc());
|
||||
} catch (IllegalAccessException | InvocationTargetException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.bypass;
|
||||
|
||||
import org.bytedeco.javacpp.FloatPointer;
|
||||
import org.bytedeco.openblas.global.openblas;
|
||||
import org.nd4j.linalg.api.blas.Level3;
|
||||
import org.nd4j.linalg.api.blas.params.GemmParams;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.factory.Nd4j;
|
||||
import org.openjdk.jmh.annotations.*;
|
||||
import org.openjdk.jmh.infra.Blackhole;
|
||||
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class BypassComparison_2x2 {
|
||||
|
||||
|
||||
@State(Scope.Thread)
|
||||
public static class SetupState {
|
||||
public int size = 2;
|
||||
public INDArray m1 = Nd4j.ones(size, size);
|
||||
public INDArray m2 = Nd4j.ones(m1.shape());
|
||||
public INDArray r = Nd4j.createUninitialized(m1.shape(), 'f');
|
||||
|
||||
public Level3 wrapper = Nd4j.getBlasWrapper().level3();
|
||||
public Method sgemm;
|
||||
public GemmParams params = new GemmParams(m1, m2, r);
|
||||
|
||||
FloatPointer a = (FloatPointer) params.getA().data().addressPointer();
|
||||
FloatPointer b = (FloatPointer) params.getB().data().addressPointer();
|
||||
FloatPointer c = (FloatPointer) params.getC().data().addressPointer();
|
||||
|
||||
@Setup(Level.Trial)
|
||||
public void doSetup(){
|
||||
try {
|
||||
sgemm = wrapper.getClass().getDeclaredMethod("sgemm", char.class, char.class, char.class, int.class, int.class, int.class, float.class, INDArray.class,
|
||||
int.class, INDArray.class, int.class, float.class, INDArray.class, int.class);
|
||||
sgemm.setAccessible(true);
|
||||
} catch (NoSuchMethodException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void sgemm(SetupState state, Blackhole bh) {
|
||||
final GemmParams params = state.params;
|
||||
try {
|
||||
state.sgemm.invoke(state.wrapper, params.getA().ordering(), params.getTransA(), params.getTransB(), params.getM(), params.getN(), params.getK(), (float)1.0, params.getA(), params.getLda(), params.getB(), params.getLdb(), (float)0.0, params.getC(), params.getLdc());
|
||||
} catch (IllegalAccessException e) {
|
||||
e.printStackTrace();
|
||||
} catch (InvocationTargetException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
public void cblas_gemm(SetupState state, Blackhole bh) {
|
||||
final GemmParams params = state.params;
|
||||
openblas.cblas_sgemm(102,111, 111, params.getM(), params.getN(), params.getK(), 1.0f, state.a, params.getLda(), state.b, params.getLdb(), 0.0f, state.c, params.getLdc());
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.bypass;
|
||||
|
||||
import org.bytedeco.javacpp.FloatPointer;
|
||||
import org.bytedeco.openblas.global.openblas;
|
||||
import org.nd4j.linalg.api.blas.Level3;
|
||||
import org.nd4j.linalg.api.blas.params.GemmParams;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.factory.Nd4j;
|
||||
import org.openjdk.jmh.annotations.*;
|
||||
import org.openjdk.jmh.infra.Blackhole;
|
||||
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class BypassComparison_8192x8192 {
|
||||
|
||||
|
||||
@State(Scope.Thread)
|
||||
public static class SetupState {
|
||||
public int size = 8192;
|
||||
public INDArray m1 = Nd4j.ones(size, size);
|
||||
public INDArray m2 = Nd4j.ones(m1.shape());
|
||||
public INDArray r = Nd4j.createUninitialized(m1.shape(), 'f');
|
||||
|
||||
public Level3 wrapper = Nd4j.getBlasWrapper().level3();
|
||||
public Method sgemm;
|
||||
public GemmParams params = new GemmParams(m1, m2, r);
|
||||
|
||||
FloatPointer a = (FloatPointer) params.getA().data().addressPointer();
|
||||
FloatPointer b = (FloatPointer) params.getB().data().addressPointer();
|
||||
FloatPointer c = (FloatPointer) params.getC().data().addressPointer();
|
||||
|
||||
@Setup(Level.Trial)
|
||||
public void doSetup(){
|
||||
try {
|
||||
sgemm = wrapper.getClass().getDeclaredMethod("sgemm", char.class, char.class, char.class, int.class, int.class, int.class, float.class, INDArray.class,
|
||||
int.class, INDArray.class, int.class, float.class, INDArray.class, int.class);
|
||||
sgemm.setAccessible(true);
|
||||
} catch (NoSuchMethodException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.SECONDS)
|
||||
public void gemm(SetupState state, Blackhole bh) {
|
||||
final GemmParams params = state.params;
|
||||
try {
|
||||
state.sgemm.invoke(state.wrapper, params.getA().ordering(), params.getTransA(), params.getTransB(), params.getM(), params.getN(), params.getK(), (float)1.0, params.getA(), params.getLda(), params.getB(), params.getLdb(), (float)0.0, params.getC(), params.getLdc());
|
||||
} catch (IllegalAccessException e) {
|
||||
e.printStackTrace();
|
||||
} catch (InvocationTargetException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.SECONDS)
|
||||
public void cblas_gemm(SetupState state, Blackhole bh) {
|
||||
final GemmParams params = state.params;
|
||||
openblas.cblas_sgemm(102,111, 111, params.getM(), params.getN(), params.getK(), 1.0f, state.a, params.getLda(), state.b, params.getLdb(), 0.0f, state.c, params.getLdc());
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package org.nd4j.memorypressure;
|
||||
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.factory.Nd4j;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
public class MemoryPressureTest {
|
||||
|
||||
public static void main(String...args) throws Exception {
|
||||
Thread[] threads = new Thread[Runtime.getRuntime().availableProcessors()];
|
||||
for (int i = 0; i < threads.length; i++) {
|
||||
INDArray arrOne = Nd4j.ones(200);
|
||||
INDArray arrTwo = Nd4j.ones(200);
|
||||
threads[i] = new Thread(() -> {
|
||||
AtomicInteger atomicInteger = new AtomicInteger(0);
|
||||
while (atomicInteger.incrementAndGet() < 100000) {
|
||||
|
||||
arrOne.addi(arrTwo);
|
||||
System.out.println("Completed " + atomicInteger.get() + " iterations");
|
||||
|
||||
}
|
||||
});
|
||||
threads[i].start();
|
||||
}
|
||||
|
||||
for (int i = 0; i < threads.length; i++) {
|
||||
threads[i].join();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
// register SLF4JBridgeHandler as handler for the j.u.l. root logger
|
||||
handlers = org.slf4j.bridge.SLF4JBridgeHandler
|
||||
@@ -0,0 +1,63 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>blas-lapack-generator</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>blas-lapack-generator</name>
|
||||
|
||||
<properties>
|
||||
<javaparser.version>3.24.4</javaparser.version>
|
||||
<maven.compiler.source>1.8</maven.compiler.source>
|
||||
<maven.compiler.target>1.8</maven.compiler.target>
|
||||
<javapoet.version>1.13.0</javapoet.version>
|
||||
<openblas.version>0.3.28</openblas.version>
|
||||
<javacpp.version>1.5.11</javacpp.version>
|
||||
<openblas.javacpp.version>${openblas.version}-${javacpp.version}</openblas.javacpp.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>nd4j-cpu-backend-common</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>nd4j-native</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.reflections</groupId>
|
||||
<artifactId>reflections</artifactId>
|
||||
<version>0.10.2</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.bytedeco</groupId>
|
||||
<artifactId>openblas</artifactId>
|
||||
<version>${openblas.javacpp.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.squareup</groupId>
|
||||
<artifactId>javapoet</artifactId>
|
||||
<version>${javapoet.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.github.javaparser</groupId>
|
||||
<artifactId>javaparser-core-serialization</artifactId>
|
||||
<version>${javaparser.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.github.javaparser</groupId>
|
||||
<artifactId>javaparser-symbol-solver-core</artifactId>
|
||||
<version>${javaparser.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
package org.deeplearning4j;
|
||||
|
||||
import com.github.javaparser.ParserConfiguration;
|
||||
import com.github.javaparser.StaticJavaParser;
|
||||
import com.github.javaparser.symbolsolver.JavaSymbolSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.JavaParserTypeSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.ReflectionTypeSolver;
|
||||
import com.github.javaparser.utils.SourceRoot;
|
||||
import com.squareup.javapoet.*;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.bytedeco.javacpp.Pointer;
|
||||
import org.bytedeco.openblas.global.openblas;
|
||||
|
||||
import javax.lang.model.element.Modifier;
|
||||
import java.io.File;
|
||||
import java.lang.reflect.Method;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
public class BlasLapackGenerator {
|
||||
|
||||
private SourceRoot sourceRoot;
|
||||
private File rootDir;
|
||||
private File targetFile;
|
||||
|
||||
private static String copyright =
|
||||
"/*\n" +
|
||||
" * ******************************************************************************\n" +
|
||||
" * *\n" +
|
||||
" * *\n" +
|
||||
" * * This program and the accompanying materials are made available under the\n" +
|
||||
" * * terms of the Apache License, Version 2.0 which is available at\n" +
|
||||
" * * https://www.apache.org/licenses/LICENSE-2.0.\n" +
|
||||
" * *\n" +
|
||||
" * * See the NOTICE file distributed with this work for additional\n" +
|
||||
" * * information regarding copyright ownership.\n" +
|
||||
" * * Unless required by applicable law or agreed to in writing, software\n" +
|
||||
" * * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n" +
|
||||
" * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n" +
|
||||
" * * License for the specific language governing permissions and limitations\n" +
|
||||
" * * under the License.\n" +
|
||||
" * *\n" +
|
||||
" * * SPDX-License-Identifier: Apache-2.0\n" +
|
||||
" * *****************************************************************************\n" +
|
||||
" */\n";
|
||||
private static String codeGenWarning =
|
||||
"\n//================== GENERATED CODE - DO NOT MODIFY THIS FILE ==================\n\n";
|
||||
|
||||
|
||||
public BlasLapackGenerator(File nd4jApiRootDir) {
|
||||
this.sourceRoot = initSourceRoot(nd4jApiRootDir);
|
||||
this.rootDir = nd4jApiRootDir;
|
||||
}
|
||||
|
||||
public SourceRoot getSourceRoot() {
|
||||
return sourceRoot;
|
||||
}
|
||||
|
||||
public void setSourceRoot(SourceRoot sourceRoot) {
|
||||
this.sourceRoot = sourceRoot;
|
||||
}
|
||||
|
||||
public File getTargetFile() {
|
||||
return targetFile;
|
||||
}
|
||||
|
||||
public void setTargetFile(File targetFile) {
|
||||
this.targetFile = targetFile;
|
||||
}
|
||||
|
||||
public void parse() throws Exception {
|
||||
targetFile = new File(rootDir,"org/nd4j/linalg/api/blas/BLASLapackDelegator.java");
|
||||
String packageName = "org.nd4j.linalg.api.blas";
|
||||
TypeSpec.Builder openblasLapackDelegator = TypeSpec.interfaceBuilder("BLASLapackDelegator");
|
||||
openblasLapackDelegator.addModifiers(Modifier.PUBLIC);
|
||||
Class<openblas> clazz = openblas.class;
|
||||
List<Method> objectMethods = Arrays.asList(Object.class.getMethods());
|
||||
Arrays.stream(clazz.getMethods())
|
||||
.filter(input -> !objectMethods.contains(input))
|
||||
.filter(input -> !input.getName().equals("map") && !input.getName().equals("init"))
|
||||
.forEach(method -> {
|
||||
MethodSpec.Builder builder = MethodSpec.methodBuilder(
|
||||
method.getName()
|
||||
).returns(method.getReturnType())
|
||||
.addModifiers(Modifier.DEFAULT,Modifier.PUBLIC);
|
||||
Arrays.stream(method.getParameters()).forEach(param -> {
|
||||
builder.addParameter(ParameterSpec.builder(
|
||||
!lapackType(param.getType()) ?
|
||||
TypeName.get(param.getType()) :
|
||||
TypeName.get(Pointer.class),
|
||||
param.getName()
|
||||
).build());
|
||||
});
|
||||
|
||||
openblasLapackDelegator.addMethod(builder.build());
|
||||
});
|
||||
|
||||
JavaFile finalFile = JavaFile.builder(packageName, openblasLapackDelegator.build())
|
||||
.addFileComment(copyright)
|
||||
.build();
|
||||
finalFile
|
||||
.writeTo(rootDir);
|
||||
}
|
||||
|
||||
private boolean lapackType(Class<?> clazz) {
|
||||
return clazz.equals(openblas.LAPACK_C_SELECT1.class) ||
|
||||
clazz.equals(openblas.LAPACK_C_SELECT2.class) ||
|
||||
clazz.equals(openblas.LAPACK_D_SELECT2.class) ||
|
||||
clazz.equals(openblas.LAPACK_S_SELECT2.class) ||
|
||||
clazz.equals(openblas.LAPACK_Z_SELECT1.class)
|
||||
|| clazz.equals(openblas.LAPACK_Z_SELECT2.class) ||
|
||||
clazz.equals(openblas.LAPACK_D_SELECT3.class) ||
|
||||
clazz.equals(openblas.LAPACK_S_SELECT3.class);
|
||||
}
|
||||
|
||||
|
||||
private SourceRoot initSourceRoot(File nd4jApiRootDir) {
|
||||
CombinedTypeSolver typeSolver = new CombinedTypeSolver();
|
||||
typeSolver.add(new ReflectionTypeSolver(false));
|
||||
typeSolver.add(new JavaParserTypeSolver(nd4jApiRootDir));
|
||||
JavaSymbolSolver symbolSolver = new JavaSymbolSolver(typeSolver);
|
||||
StaticJavaParser.getConfiguration().setSymbolResolver(symbolSolver);
|
||||
SourceRoot sourceRoot = new SourceRoot(nd4jApiRootDir.toPath(),new ParserConfiguration().setSymbolResolver(symbolSolver));
|
||||
return sourceRoot;
|
||||
}
|
||||
|
||||
|
||||
public static void main(String...args) throws Exception {
|
||||
BlasLapackGenerator blasLapackGenerator = new BlasLapackGenerator(new File("../../nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/"));
|
||||
blasLapackGenerator.parse();
|
||||
String generated = FileUtils.readFileToString(blasLapackGenerator.getTargetFile(), Charset.defaultCharset());
|
||||
generated = generated.replaceAll("\\{\\s+\\}",";");
|
||||
generated = generated.replace("default","");
|
||||
FileUtils.write(blasLapackGenerator.getTargetFile(),generated);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
package org.deeplearning4j;
|
||||
|
||||
import com.github.javaparser.ParserConfiguration;
|
||||
import com.github.javaparser.StaticJavaParser;
|
||||
import com.github.javaparser.symbolsolver.JavaSymbolSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.JavaParserTypeSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.ReflectionTypeSolver;
|
||||
import com.github.javaparser.utils.SourceRoot;
|
||||
import com.squareup.javapoet.*;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.nd4j.linalg.api.blas.BLASLapackDelegator;
|
||||
|
||||
import javax.lang.model.element.Modifier;
|
||||
import java.io.File;
|
||||
import java.lang.reflect.Method;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
public class CudaBlasLapackGenerator {
|
||||
|
||||
private SourceRoot sourceRoot;
|
||||
private File rootDir;
|
||||
private File targetFile;
|
||||
|
||||
|
||||
private static String copyright =
|
||||
"/*\n" +
|
||||
" * ******************************************************************************\n" +
|
||||
" * *\n" +
|
||||
" * *\n" +
|
||||
" * * This program and the accompanying materials are made available under the\n" +
|
||||
" * * terms of the Apache License, Version 2.0 which is available at\n" +
|
||||
" * * https://www.apache.org/licenses/LICENSE-2.0.\n" +
|
||||
" * *\n" +
|
||||
" * * See the NOTICE file distributed with this work for additional\n" +
|
||||
" * * information regarding copyright ownership.\n" +
|
||||
" * * Unless required by applicable law or agreed to in writing, software\n" +
|
||||
" * * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n" +
|
||||
" * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n" +
|
||||
" * * License for the specific language governing permissions and limitations\n" +
|
||||
" * * under the License.\n" +
|
||||
" * *\n" +
|
||||
" * * SPDX-License-Identifier: Apache-2.0\n" +
|
||||
" * *****************************************************************************\n" +
|
||||
" */\n";
|
||||
private static String codeGenWarning =
|
||||
"\n//================== GENERATED CODE - DO NOT MODIFY THIS FILE ==================\n\n";
|
||||
|
||||
|
||||
public CudaBlasLapackGenerator(File nd4jApiRootDir) {
|
||||
this.sourceRoot = initSourceRoot(nd4jApiRootDir);
|
||||
this.rootDir = nd4jApiRootDir;
|
||||
}
|
||||
|
||||
|
||||
public void parse() throws Exception {
|
||||
targetFile = new File(rootDir,"org/nd4j/linalg/jcublas/blas/CudaBLASDelegator.java");
|
||||
String packageName = "org.nd4j.linalg.jcublas.blas";
|
||||
TypeSpec.Builder openblasLapackDelegator = TypeSpec.classBuilder("CudaBLASDelegator");
|
||||
openblasLapackDelegator.addModifiers(Modifier.PUBLIC);
|
||||
openblasLapackDelegator.addSuperinterface(BLASLapackDelegator.class);
|
||||
|
||||
Class<BLASLapackDelegator> clazz = BLASLapackDelegator.class;
|
||||
List<Method> objectMethods = Arrays.asList(Object.class.getMethods());
|
||||
Set<MethodSpec> addedCodeLines = new HashSet<>();
|
||||
Arrays.stream(clazz.getMethods())
|
||||
.filter(input -> !objectMethods.contains(input))
|
||||
.forEach(method -> {
|
||||
MethodSpec.Builder builder = MethodSpec.methodBuilder(
|
||||
method.getName()
|
||||
).addModifiers(Modifier.PUBLIC)
|
||||
.returns(method.getReturnType())
|
||||
.addAnnotation(Override.class);
|
||||
StringBuilder codeStatement = new StringBuilder();
|
||||
//don't return anything when void
|
||||
if(method.getReturnType().equals(Void.TYPE)) {
|
||||
|
||||
} else if(method.getReturnType().equals(int.class)){
|
||||
codeStatement.append("return 0;");
|
||||
|
||||
} else if(method.getReturnType().equals(double.class)) {
|
||||
codeStatement.append("return 0.0;");
|
||||
|
||||
} else if(method.getReturnType().equals(float.class)) {
|
||||
codeStatement.append("return 0.0f;");
|
||||
|
||||
}
|
||||
else if(method.getReturnType().equals(long.class)) {
|
||||
codeStatement.append("return 0L;");
|
||||
}
|
||||
|
||||
Arrays.stream(method.getParameters()).forEach(param -> {
|
||||
builder.addParameter(ParameterSpec.builder(param.getType(),param.getName())
|
||||
.build());
|
||||
|
||||
});
|
||||
|
||||
|
||||
builder.addCode(CodeBlock
|
||||
.builder()
|
||||
.addStatement(codeStatement.toString().replace(",)",")"))
|
||||
.build());
|
||||
|
||||
MethodSpec build = builder.build();
|
||||
openblasLapackDelegator.addMethod(build);
|
||||
addedCodeLines.add(build);
|
||||
|
||||
|
||||
});
|
||||
|
||||
JavaFile.builder(packageName,openblasLapackDelegator.build())
|
||||
.addFileComment(copyright)
|
||||
.build()
|
||||
.writeTo(rootDir);
|
||||
}
|
||||
|
||||
|
||||
private SourceRoot initSourceRoot(File nd4jApiRootDir) {
|
||||
CombinedTypeSolver typeSolver = new CombinedTypeSolver();
|
||||
typeSolver.add(new ReflectionTypeSolver(false));
|
||||
typeSolver.add(new JavaParserTypeSolver(nd4jApiRootDir));
|
||||
JavaSymbolSolver symbolSolver = new JavaSymbolSolver(typeSolver);
|
||||
StaticJavaParser.getConfiguration().setSymbolResolver(symbolSolver);
|
||||
SourceRoot sourceRoot = new SourceRoot(nd4jApiRootDir.toPath(),new ParserConfiguration().setSymbolResolver(symbolSolver));
|
||||
return sourceRoot;
|
||||
}
|
||||
|
||||
public SourceRoot getSourceRoot() {
|
||||
return sourceRoot;
|
||||
}
|
||||
|
||||
public File getRootDir() {
|
||||
return rootDir;
|
||||
}
|
||||
|
||||
public File getTargetFile() {
|
||||
return targetFile;
|
||||
}
|
||||
|
||||
public static void main(String...args) throws Exception {
|
||||
CudaBlasLapackGenerator cudaBlasLapackGenerator = new CudaBlasLapackGenerator(new File("../../nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java"));
|
||||
cudaBlasLapackGenerator.parse();
|
||||
String generated = FileUtils.readFileToString(cudaBlasLapackGenerator.getTargetFile(), Charset.defaultCharset());
|
||||
generated = generated.replace(";;",";");
|
||||
FileUtils.write(cudaBlasLapackGenerator.getTargetFile(),generated);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
package org.deeplearning4j;
|
||||
|
||||
import com.github.javaparser.ParserConfiguration;
|
||||
import com.github.javaparser.StaticJavaParser;
|
||||
import com.github.javaparser.symbolsolver.JavaSymbolSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.JavaParserTypeSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.ReflectionTypeSolver;
|
||||
import com.github.javaparser.utils.SourceRoot;
|
||||
import com.squareup.javapoet.*;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.bytedeco.javacpp.Pointer;
|
||||
import org.bytedeco.openblas.global.openblas;
|
||||
import org.bytedeco.openblas.global.openblas_nolapack;
|
||||
import org.nd4j.linalg.api.blas.BLASLapackDelegator;
|
||||
|
||||
import javax.lang.model.element.Modifier;
|
||||
import java.io.File;
|
||||
import java.lang.reflect.Method;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.*;
|
||||
|
||||
public class NoOpBlasLapackGenerator {
|
||||
|
||||
private SourceRoot sourceRoot;
|
||||
private File rootDir;
|
||||
private File targetFile;
|
||||
|
||||
|
||||
private static String copyright =
|
||||
"/*\n" +
|
||||
" * ******************************************************************************\n" +
|
||||
" * *\n" +
|
||||
" * *\n" +
|
||||
" * * This program and the accompanying materials are made available under the\n" +
|
||||
" * * terms of the Apache License, Version 2.0 which is available at\n" +
|
||||
" * * https://www.apache.org/licenses/LICENSE-2.0.\n" +
|
||||
" * *\n" +
|
||||
" * * See the NOTICE file distributed with this work for additional\n" +
|
||||
" * * information regarding copyright ownership.\n" +
|
||||
" * * Unless required by applicable law or agreed to in writing, software\n" +
|
||||
" * * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n" +
|
||||
" * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n" +
|
||||
" * * License for the specific language governing permissions and limitations\n" +
|
||||
" * * under the License.\n" +
|
||||
" * *\n" +
|
||||
" * * SPDX-License-Identifier: Apache-2.0\n" +
|
||||
" * *****************************************************************************\n" +
|
||||
" */\n";
|
||||
private static String codeGenWarning =
|
||||
"\n//================== GENERATED CODE - DO NOT MODIFY THIS FILE ==================\n\n";
|
||||
|
||||
|
||||
public NoOpBlasLapackGenerator(File nd4jApiRootDir) {
|
||||
this.sourceRoot = initSourceRoot(nd4jApiRootDir);
|
||||
this.rootDir = nd4jApiRootDir;
|
||||
}
|
||||
|
||||
|
||||
public void parse() throws Exception {
|
||||
targetFile = new File(rootDir,"org/nd4j/linalg/cpu/nativecpu/blas/NoOpBLASDelegator.java");
|
||||
String packageName = "org.nd4j.linalg.cpu.nativecpu.blas";
|
||||
TypeSpec.Builder openblasLapackDelegator = TypeSpec.classBuilder("NoOpBLASDelegator");
|
||||
openblasLapackDelegator.addModifiers(Modifier.PUBLIC);
|
||||
openblasLapackDelegator.addSuperinterface(BLASLapackDelegator.class);
|
||||
|
||||
Class<BLASLapackDelegator> clazz = BLASLapackDelegator.class;
|
||||
List<Method> objectMethods = Arrays.asList(Object.class.getMethods());
|
||||
Set<MethodSpec> addedCodeLines = new HashSet<>();
|
||||
Arrays.stream(clazz.getMethods())
|
||||
.filter(input -> !objectMethods.contains(input))
|
||||
.forEach(method -> {
|
||||
MethodSpec.Builder builder = MethodSpec.methodBuilder(
|
||||
method.getName()
|
||||
).addModifiers(Modifier.PUBLIC)
|
||||
.returns(method.getReturnType())
|
||||
.addAnnotation(Override.class);
|
||||
StringBuilder codeStatement = new StringBuilder();
|
||||
//don't return anything when void
|
||||
if(method.getReturnType().equals(Void.TYPE)) {
|
||||
|
||||
} else if(method.getReturnType().equals(int.class)){
|
||||
codeStatement.append("return 0;");
|
||||
|
||||
} else if(method.getReturnType().equals(double.class)) {
|
||||
codeStatement.append("return 0.0;");
|
||||
|
||||
} else if(method.getReturnType().equals(float.class)) {
|
||||
codeStatement.append("return 0.0f;");
|
||||
|
||||
}
|
||||
else if(method.getReturnType().equals(long.class)) {
|
||||
codeStatement.append("return 0L;");
|
||||
}
|
||||
|
||||
Arrays.stream(method.getParameters()).forEach(param -> {
|
||||
builder.addParameter(ParameterSpec.builder(param.getType(),param.getName())
|
||||
.build());
|
||||
|
||||
});
|
||||
|
||||
|
||||
builder.addCode(CodeBlock
|
||||
.builder()
|
||||
.addStatement(codeStatement.toString().replace(",)",")"))
|
||||
.build());
|
||||
|
||||
MethodSpec build = builder.build();
|
||||
openblasLapackDelegator.addMethod(build);
|
||||
addedCodeLines.add(build);
|
||||
|
||||
|
||||
});
|
||||
|
||||
JavaFile.builder(packageName,openblasLapackDelegator.build())
|
||||
.addFileComment(copyright)
|
||||
.build()
|
||||
.writeTo(rootDir);
|
||||
}
|
||||
|
||||
|
||||
private SourceRoot initSourceRoot(File nd4jApiRootDir) {
|
||||
CombinedTypeSolver typeSolver = new CombinedTypeSolver();
|
||||
typeSolver.add(new ReflectionTypeSolver(false));
|
||||
typeSolver.add(new JavaParserTypeSolver(nd4jApiRootDir));
|
||||
JavaSymbolSolver symbolSolver = new JavaSymbolSolver(typeSolver);
|
||||
StaticJavaParser.getConfiguration().setSymbolResolver(symbolSolver);
|
||||
SourceRoot sourceRoot = new SourceRoot(nd4jApiRootDir.toPath(),new ParserConfiguration().setSymbolResolver(symbolSolver));
|
||||
return sourceRoot;
|
||||
}
|
||||
|
||||
public SourceRoot getSourceRoot() {
|
||||
return sourceRoot;
|
||||
}
|
||||
|
||||
public File getRootDir() {
|
||||
return rootDir;
|
||||
}
|
||||
|
||||
public File getTargetFile() {
|
||||
return targetFile;
|
||||
}
|
||||
|
||||
public static void main(String...args) throws Exception {
|
||||
NoOpBlasLapackGenerator openblasBlasLapackGenerator = new NoOpBlasLapackGenerator(new File("../../nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cpu-backend-common/src/main/java"));
|
||||
openblasBlasLapackGenerator.parse();
|
||||
String generated = FileUtils.readFileToString(openblasBlasLapackGenerator.getTargetFile(), Charset.defaultCharset());
|
||||
generated = generated.replace(";;",";");
|
||||
FileUtils.write(openblasBlasLapackGenerator.getTargetFile(),generated);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+268
@@ -0,0 +1,268 @@
|
||||
package org.deeplearning4j;
|
||||
|
||||
import com.github.javaparser.ParserConfiguration;
|
||||
import com.github.javaparser.StaticJavaParser;
|
||||
import com.github.javaparser.symbolsolver.JavaSymbolSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.JavaParserTypeSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.ReflectionTypeSolver;
|
||||
import com.github.javaparser.utils.SourceRoot;
|
||||
import com.squareup.javapoet.*;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.bytedeco.javacpp.Pointer;
|
||||
import org.bytedeco.openblas.global.openblas;
|
||||
import org.bytedeco.openblas.global.openblas_nolapack;
|
||||
import org.nd4j.linalg.api.blas.BLASLapackDelegator;
|
||||
import org.nd4j.shade.guava.collect.HashBasedTable;
|
||||
import org.nd4j.shade.guava.collect.Table;
|
||||
|
||||
import javax.lang.model.element.Modifier;
|
||||
import java.io.File;
|
||||
import java.lang.reflect.Method;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.*;
|
||||
|
||||
public class OpenblasBlasLapackGenerator {
|
||||
|
||||
private SourceRoot sourceRoot;
|
||||
private File rootDir;
|
||||
private File targetFile;
|
||||
|
||||
private Map<String,String> casting = new HashMap<String,String>(){{
|
||||
put("LAPACKE_sgees","openblas.LAPACK_S_SELECT2");
|
||||
put("LAPACKE_dgees","openblas.LAPACK_D_SELECT2");
|
||||
put("LAPACKE_cgees","openblas.LAPACK_C_SELECT1");
|
||||
put("LAPACKE_zgees","openblas.LAPACK_Z_SELECT1");
|
||||
put("LAPACKE_sgeesx","openblas.LAPACK_S_SELECT2");
|
||||
put("LAPACKE_dgeesx","openblas.LAPACK_D_SELECT2");
|
||||
put("LAPACKE_cgeesx","openblas.LAPACK_C_SELECT1");
|
||||
put("LAPACKE_zgeesx","openblas.LAPACK_Z_SELECT1");
|
||||
put("LAPACKE_sgges","openblas.LAPACK_S_SELECT3");
|
||||
put("LAPACKE_dgges","openblas.LAPACK_D_SELECT3");
|
||||
put("LAPACKE_cgges","openblas.LAPACK_C_SELECT2");
|
||||
put("LAPACKE_zgges","openblas.LAPACK_Z_SELECT2");
|
||||
put("LAPACKE_sgges3","openblas.LAPACK_S_SELECT3");
|
||||
put("LAPACKE_dgges3","openblas.LAPACK_D_SELECT3");
|
||||
put("LAPACKE_cgges3","openblas.LAPACK_C_SELECT2");
|
||||
put("LAPACKE_zgges3","openblas.LAPACK_Z_SELECT2");
|
||||
put("LAPACKE_sggesx","openblas.LAPACK_S_SELECT3");
|
||||
put("LAPACKE_dggesx","openblas.LAPACK_D_SELECT3");
|
||||
put("LAPACKE_cggesx","openblas.LAPACK_C_SELECT2");
|
||||
put("LAPACKE_zggesx","openblas.LAPACK_Z_SELECT2");
|
||||
put("LAPACKE_sgees_work","openblas.LAPACK_S_SELECT2");
|
||||
put("LAPACKE_dgees_work","openblas.LAPACK_D_SELECT2");
|
||||
put("LAPACKE_cgees_work","openblas.LAPACK_C_SELECT1");
|
||||
put("LAPACKE_zgees_work","openblas.LAPACK_Z_SELECT1");
|
||||
put("LAPACKE_sgeesx_work","openblas.LAPACK_S_SELECT2");
|
||||
put("LAPACKE_dgeesx_work","openblas.LAPACK_D_SELECT2");
|
||||
put("LAPACKE_cgeesx_work","openblas.LAPACK_C_SELECT1");
|
||||
put("LAPACKE_zgeesx_work","openblas.LAPACK_Z_SELECT1");
|
||||
put("LAPACKE_sgges_work","openblas.LAPACK_S_SELECT3");
|
||||
put("LAPACKE_dgges_work","openblas.LAPACK_D_SELECT3");
|
||||
put("LAPACKE_cgges_work","openblas.LAPACK_C_SELECT2");
|
||||
put("LAPACKE_zgges_work","openblas.LAPACK_Z_SELECT2");
|
||||
put("LAPACKE_sgges3_work","openblas.LAPACK_S_SELECT3");
|
||||
put("LAPACKE_dgges3_work","openblas.LAPACK_D_SELECT3");
|
||||
put("LAPACKE_cgges3_work","openblas.LAPACK_C_SELECT2");
|
||||
put("LAPACKE_zgges3_work","openblas.LAPACK_Z_SELECT2");
|
||||
put("LAPACKE_sggesx_work","openblas.LAPACK_S_SELECT3");
|
||||
put("LAPACKE_dggesx_work","openblas.LAPACK_D_SELECT3");
|
||||
put("LAPACKE_cggesx_work","openblas.LAPACK_C_SELECT2");
|
||||
put("LAPACKE_zggesx_work","openblas.LAPACK_Z_SELECT2");
|
||||
|
||||
put("LAPACK_sgges3","openblas.LAPACK_S_SELECT3");
|
||||
put("LAPACK_dgges3","openblas.LAPACK_D_SELECT3");
|
||||
put("LAPACK_cgges3","openblas.LAPACK_C_SELECT2");
|
||||
put("LAPACK_zgges3","openblas.LAPACK_Z_SELECT2");
|
||||
|
||||
|
||||
put("LAPACK_sgges","openblas.LAPACK_S_SELECT3");
|
||||
put("LAPACK_dgges","openblas.LAPACK_D_SELECT3");
|
||||
put("LAPACK_cgges","openblas.LAPACK_C_SELECT2");
|
||||
put("LAPACK_zgges","openblas.LAPACK_Z_SELECT2");
|
||||
|
||||
|
||||
put("LAPACK_sggesx","openblas.LAPACK_S_SELECT3");
|
||||
put("LAPACK_dggesx","openblas.LAPACK_D_SELECT3");
|
||||
put("LAPACK_cggesx","openblas.LAPACK_C_SELECT2");
|
||||
put("LAPACK_zggesx","openblas.LAPACK_Z_SELECT2");
|
||||
|
||||
//LAPACK_zgeesx
|
||||
put("LAPACK_cgees","openblas.LAPACK_C_SELECT1");
|
||||
put("LAPACK_dgees","openblas.LAPACK_D_SELECT2");
|
||||
put("LAPACK_zgees","openblas.LAPACK_Z_SELECT1");
|
||||
put("LAPACK_sgees","openblas.LAPACK_S_SELECT2");
|
||||
|
||||
|
||||
put("LAPACK_cgeesx","openblas.LAPACK_C_SELECT1");
|
||||
put("LAPACK_dgeesx","openblas.LAPACK_D_SELECT2");
|
||||
put("LAPACK_zgeesx","openblas.LAPACK_Z_SELECT1");
|
||||
put("LAPACK_sgeesx","openblas.LAPACK_S_SELECT2");
|
||||
|
||||
}};
|
||||
private static String copyright =
|
||||
"/*\n" +
|
||||
" * ******************************************************************************\n" +
|
||||
" * *\n" +
|
||||
" * *\n" +
|
||||
" * * This program and the accompanying materials are made available under the\n" +
|
||||
" * * terms of the Apache License, Version 2.0 which is available at\n" +
|
||||
" * * https://www.apache.org/licenses/LICENSE-2.0.\n" +
|
||||
" * *\n" +
|
||||
" * * See the NOTICE file distributed with this work for additional\n" +
|
||||
" * * information regarding copyright ownership.\n" +
|
||||
" * * Unless required by applicable law or agreed to in writing, software\n" +
|
||||
" * * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n" +
|
||||
" * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n" +
|
||||
" * * License for the specific language governing permissions and limitations\n" +
|
||||
" * * under the License.\n" +
|
||||
" * *\n" +
|
||||
" * * SPDX-License-Identifier: Apache-2.0\n" +
|
||||
" * *****************************************************************************\n" +
|
||||
" */\n";
|
||||
private static String codeGenWarning =
|
||||
"\n//================== GENERATED CODE - DO NOT MODIFY THIS FILE ==================\n\n";
|
||||
|
||||
|
||||
public OpenblasBlasLapackGenerator(File nd4jApiRootDir) {
|
||||
this.sourceRoot = initSourceRoot(nd4jApiRootDir);
|
||||
this.rootDir = nd4jApiRootDir;
|
||||
}
|
||||
|
||||
public void parse() throws Exception {
|
||||
targetFile = new File(rootDir,"org/nd4j/linalg/cpu/nativecpu/OpenblasLapackDelegator.java");
|
||||
String packageName = "org.nd4j.linalg.cpu.nativecpu";
|
||||
TypeSpec.Builder openblasLapackDelegator = TypeSpec.classBuilder("OpenblasLapackDelegator");
|
||||
openblasLapackDelegator.addModifiers(Modifier.PUBLIC);
|
||||
openblasLapackDelegator.addSuperinterface(BLASLapackDelegator.class);
|
||||
|
||||
Class<BLASLapackDelegator> clazz = BLASLapackDelegator.class;
|
||||
List<Method> objectMethods = Arrays.asList(Object.class.getMethods());
|
||||
Set<MethodSpec> addedCodeLines = new HashSet<>();
|
||||
Arrays.stream(clazz.getMethods())
|
||||
.filter(input -> !objectMethods.contains(input))
|
||||
.forEach(method -> {
|
||||
MethodSpec.Builder builder = MethodSpec.methodBuilder(
|
||||
method.getName()
|
||||
).addModifiers(Modifier.PUBLIC)
|
||||
.returns(method.getReturnType())
|
||||
.addAnnotation(Override.class);
|
||||
StringBuilder codeStatement = new StringBuilder();
|
||||
//don't return anything when void
|
||||
if(method.getReturnType().equals(Void.TYPE)) {
|
||||
codeStatement.append("openblas." + method.getName() + "(");
|
||||
|
||||
} else if(method.getReturnType().equals(int.class)){
|
||||
//codeStatement.append("return 0;");
|
||||
codeStatement.append("return openblas." + method.getName() + "(");
|
||||
|
||||
} else if(method.getReturnType().equals(double.class)) {
|
||||
//codeStatement.append("return 0.0;");
|
||||
codeStatement.append("return openblas." + method.getName() + "(");
|
||||
|
||||
} else if(method.getReturnType().equals(float.class)) {
|
||||
//codeStatement.append("return 0.0f;");
|
||||
codeStatement.append("return openblas." + method.getName() + "(");
|
||||
|
||||
}
|
||||
else if(method.getReturnType().equals(long.class)) {
|
||||
//codeStatement.append("return 0L;");
|
||||
codeStatement.append("return openblas." + method.getName() + "(");
|
||||
|
||||
}
|
||||
|
||||
//TODO: LAPACK_cgees
|
||||
//TODO: LAPACK_dgees
|
||||
//TODO: LAPACK_zgees
|
||||
//TODO: LAPACK_cgeesx
|
||||
//TODO: LAPACK_dgeesx
|
||||
//TODO: LAPACK_sgeesx
|
||||
//TODO: LAPACK_zgeesx
|
||||
//TODO: LAPACK_cgges
|
||||
//TODO: LAPACK_dgges
|
||||
//TODO: LAPACK_sgges
|
||||
//TODO: LAPACK_zgges
|
||||
//TODO: LAPACK_cgges3
|
||||
//TODO: LAPACK_dgges3
|
||||
//TODO: LAPACK_sgges3
|
||||
//TODO: LAPACK_zgges3
|
||||
//TODO: LAPACK_cggesx
|
||||
//TODO: LAPACK_dggesx
|
||||
//TODO: LAPACK_sggesx
|
||||
//TODO: LAPACK_zggesx
|
||||
|
||||
|
||||
//TODO: issue could be LAPACK_Z_SELECT_2
|
||||
//TODO: LAPACK_S_SELECT_3
|
||||
Arrays.stream(method.getParameters()).forEach(param -> {
|
||||
if(casting.containsKey(method.getName()) && param.getType().equals(Pointer.class)) {
|
||||
System.out.println("In function casting for " + method.getName());
|
||||
codeStatement.append("((" + casting.get(method.getName()) + ")" + param.getName() + ")");
|
||||
codeStatement.append(",");
|
||||
} else {
|
||||
codeStatement.append(param.getName());
|
||||
codeStatement.append(",");
|
||||
}
|
||||
|
||||
builder.addParameter(ParameterSpec.builder(param.getType(),param.getName())
|
||||
.build());
|
||||
|
||||
});
|
||||
|
||||
codeStatement.append(")");
|
||||
|
||||
builder.addCode(CodeBlock
|
||||
.builder()
|
||||
.addStatement(codeStatement.toString().replace(",)",")"))
|
||||
.build());
|
||||
|
||||
MethodSpec build = builder.build();
|
||||
openblasLapackDelegator.addMethod(build);
|
||||
addedCodeLines.add(build);
|
||||
|
||||
|
||||
});
|
||||
|
||||
JavaFile.builder(packageName,openblasLapackDelegator.build())
|
||||
.addFileComment(copyright)
|
||||
.addStaticImport(openblas.class,"*")
|
||||
.addStaticImport(openblas_nolapack.class,"*")
|
||||
.build()
|
||||
.writeTo(rootDir);
|
||||
}
|
||||
|
||||
|
||||
private SourceRoot initSourceRoot(File nd4jApiRootDir) {
|
||||
CombinedTypeSolver typeSolver = new CombinedTypeSolver();
|
||||
typeSolver.add(new ReflectionTypeSolver(false));
|
||||
typeSolver.add(new JavaParserTypeSolver(nd4jApiRootDir));
|
||||
JavaSymbolSolver symbolSolver = new JavaSymbolSolver(typeSolver);
|
||||
StaticJavaParser.getConfiguration().setSymbolResolver(symbolSolver);
|
||||
SourceRoot sourceRoot = new SourceRoot(nd4jApiRootDir.toPath(),new ParserConfiguration().setSymbolResolver(symbolSolver));
|
||||
return sourceRoot;
|
||||
}
|
||||
|
||||
public SourceRoot getSourceRoot() {
|
||||
return sourceRoot;
|
||||
}
|
||||
|
||||
public File getRootDir() {
|
||||
return rootDir;
|
||||
}
|
||||
|
||||
public File getTargetFile() {
|
||||
return targetFile;
|
||||
}
|
||||
|
||||
public static void main(String...args) throws Exception {
|
||||
OpenblasBlasLapackGenerator openblasBlasLapackGenerator = new OpenblasBlasLapackGenerator(new File("../../nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java"));
|
||||
openblasBlasLapackGenerator.parse();
|
||||
String generated = FileUtils.readFileToString(openblasBlasLapackGenerator.getTargetFile(), Charset.defaultCharset());
|
||||
generated = generated.replace(";;",";");
|
||||
generated = generated.replaceAll("import static org.bytedeco.openblas.global.openblas\\.\\*","import org.bytedeco.openblas.global.openblas");
|
||||
generated = generated.replaceAll("import static org.bytedeco.openblas.global.openblas_nolapack\\.\\*","import org.bytedeco.openblas.global.openblas_nolapack");
|
||||
FileUtils.write(openblasBlasLapackGenerator.getTargetFile(),generated,Charset.defaultCharset());
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
@@ -0,0 +1,84 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
|
||||
http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.deeplearning4j.tools</groupId>
|
||||
<artifactId>cpp-dependency-analyzer</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>C++ Dependency Analyzer</name>
|
||||
<description>Tool to analyze C++ include dependencies within deeplearning4j/libnd4j</description>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<picocli.version>4.7.5</picocli.version>
|
||||
<junit.version>5.10.0</junit.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>info.picocli</groupId>
|
||||
<artifactId>picocli</artifactId>
|
||||
<version>${picocli.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<version>${junit.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.11.0</version>
|
||||
<configuration>
|
||||
<source>11</source>
|
||||
<target>11</target>
|
||||
<annotationProcessorPaths>
|
||||
<path>
|
||||
<groupId>info.picocli</groupId>
|
||||
<artifactId>picocli-codegen</artifactId>
|
||||
<version>${picocli.version}</version>
|
||||
</path>
|
||||
</annotationProcessorPaths>
|
||||
<compilerArgs>
|
||||
<arg>-Aproject=${project.groupId}/${project.artifactId}</arg>
|
||||
</compilerArgs>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>3.4.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<createDependencyReducedPom>false</createDependencyReducedPom>
|
||||
<transformers>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
|
||||
<mainClass>org.deeplearning4j.tools.CppDependencyAnalyzer</mainClass>
|
||||
</transformer>
|
||||
</transformers>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
@@ -0,0 +1,357 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!--
|
||||
~ /* ******************************************************************************
|
||||
~ *
|
||||
~ *
|
||||
~ * This program and the accompanying materials are made available under the
|
||||
~ * terms of the Apache License, Version 2.0 which is available at
|
||||
~ * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
~ *
|
||||
~ * See the NOTICE file distributed with this work for additional
|
||||
~ * information regarding copyright ownership.
|
||||
~ * Unless required by applicable law or agreed to in writing, software
|
||||
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
~ * License for the specific language governing permissions and limitations
|
||||
~ * under the License.
|
||||
~ *
|
||||
~ * SPDX-License-Identifier: Apache-2.0
|
||||
~ ******************************************************************************/
|
||||
-->
|
||||
|
||||
<profiles version="13">
|
||||
<profile kind="CodeFormatterProfile" name="GoogleStyle" version="13">
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.disabling_tag" value="@formatter:off"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration" value="end_of_line"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_cascading_method_invocation_with_arguments.count_dependent" value="16|-1|16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_field" value="0"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.use_on_off_tags" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.wrap_prefer_two_fragments" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_ellipsis" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.wrap_comment_inline_tags" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_local_variable_declaration" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiple_fields" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_parameter" value="1040"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_type.count_dependent" value="1585|-1|1585"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression" value="80"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiple_fields.count_dependent" value="16|-1|16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_binary_operator" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_array_initializer" value="end_of_line"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_package" value="1"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression.count_dependent" value="16|4|80"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration.count_dependent" value="16|4|48"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.continuation_indentation" value="4"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration.count_dependent" value="16|4|49"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk" value="1"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_binary_operator" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_package" value="0"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_cascading_method_invocation_with_arguments" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.compiler.source" value="1.7"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration.count_dependent" value="16|4|48"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.format_line_comments" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.join_wrapped_lines" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.wrap_non_simple_local_variable_annotation" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.align_type_members_on_columns" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_member_type" value="0"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_enum_constants.count_dependent" value="16|5|48"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_unary_operator" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.indent_parameter_description" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.lineSplit" value="120"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation.count_dependent" value="16|4|48"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration" value="0"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.enabling_tag" value="@formatter:on"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_package" value="1585"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_assignment" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.compiler.problem.assertIdentifier" value="error"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.tabulation.char" value="space"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_body" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_method" value="1"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.wrap_non_simple_type_annotation" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_field_declaration" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration" value="end_of_line"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_method_declaration" value="0"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_switch" value="end_of_line"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.compiler.problem.enumIdentifier" value="error"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_generic_type_arguments" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment_new_line_at_start_of_html_paragraph" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_ellipsis" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block" value="end_of_line"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comment_prefix" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_method_declaration" value="end_of_line"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.compact_else_if" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.wrap_non_simple_parameter_annotation" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_method" value="1585"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.indent_root_tags" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_constant" value="end_of_line"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.tabulation.size" value="4"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation.count_dependent" value="16|5|80"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_parameter.count_dependent" value="1040|-1|1040"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_package.count_dependent" value="1585|-1|1585"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.indent_empty_lines" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.force_if_else_statement_brace" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block_in_case" value="end_of_line"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve" value="3"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.wrap_non_simple_package_annotation" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation.count_dependent" value="16|-1|16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_type" value="1585"/>
|
||||
<setting id="org.eclipse.jdt.core.compiler.compliance" value="1.7"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer" value="4"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_unary_operator" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_new_anonymous_class" value="20"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_local_variable.count_dependent" value="1585|-1|1585"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_field.count_dependent" value="1585|-1|1585"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration.count_dependent" value="16|5|80"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration" value="end_of_line"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_binary_expression" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode" value="enabled"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_label" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant.count_dependent" value="16|-1|16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.line_length" value="120"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_import_groups" value="1"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration" value="end_of_line"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body" value="0"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.wrap_before_binary_operator" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations" value="2"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_block" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.join_lines_in_comments" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_if" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_imports" value="0"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_field" value="1585"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.format_html" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer.count_dependent" value="16|5|80"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.format_source_code" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.compiler.codegen.targetPlatform" value="1.7"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_resources_in_try" value="80"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.format_header" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.format_block_comments" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_enum_constants" value="0"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration.count_dependent" value="16|4|48"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_method.count_dependent" value="1585|-1|1585"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_type_declaration" value="end_of_line"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_binary_expression.count_dependent" value="16|-1|16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.wrap_non_simple_member_annotation" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_local_variable" value="1585"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call.count_dependent" value="16|5|80"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_generic_type_arguments.count_dependent" value="16|-1|16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression.count_dependent" value="16|5|80"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration.count_dependent" value="16|5|80"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_imports" value="1"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header" value="true"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for" value="insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.alignment_for_for_statement" value="16"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments" value="do not insert"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column" value="false"/>
|
||||
<setting id="org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line" value="false"/>
|
||||
</profile>
|
||||
</profiles>
|
||||
@@ -0,0 +1,220 @@
|
||||
# ND4J Log Analyzer
|
||||
|
||||
## Overview
|
||||
|
||||
ND4J Log Analyzer is a Java agent designed to record ND4J operation executions in an H2 database and index a specified DeepLearning4J codebase. This tool is crucial for identifying regressions between different versions of DeepLearning4J by analyzing the execution patterns and performance metrics of ND4J operations.
|
||||
|
||||
## Features
|
||||
|
||||
- Records ND4J operation executions in real-time
|
||||
- Stores execution data in an H2 database for efficient querying
|
||||
- Indexes the specified DeepLearning4J codebase for reference
|
||||
- Can be injected into a running DeepLearning4J application as a Java agent
|
||||
- Provides methods for querying and analyzing recorded operations
|
||||
- Includes a StackTraceCodeFinder utility for locating source code lines
|
||||
- Features a JsonComparisonReport tool for comparing operation logs between different runs or versions
|
||||
- Offers a JsonReport utility for exporting operation logs to JSON format
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Java 8 or higher
|
||||
- Maven 3.x
|
||||
- Access to the DeepLearning4J codebase you want to analyze
|
||||
|
||||
## Installation
|
||||
|
||||
1. Clone the repository:
|
||||
```
|
||||
git clone https://github.com/your-repo/nd4j-log-analyzer.git
|
||||
```
|
||||
|
||||
2. Navigate to the project directory:
|
||||
```
|
||||
cd nd4j-log-analyzer
|
||||
```
|
||||
|
||||
3. Build the project using Maven:
|
||||
```
|
||||
mvn clean package
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
To use the ND4J Log Analyzer, you need to inject it as a Java agent into your DeepLearning4J application. Use the following VM arguments when running your Java application:
|
||||
|
||||
```
|
||||
-DsourceCodeIndexerPath=/path/to/your/deeplearning4j/codebase -javaagent:/path/to/nd4j-log-analyzer-1.0-SNAPSHOT.jar
|
||||
```
|
||||
|
||||
Example:
|
||||
```
|
||||
-DsourceCodeIndexerPath=/home/user/Documents/GitHub/deeplearning4j/ -javaagent:/home/user/Documents/GitHub/deeplearning4j/contrib/nd4j-log-analyzer/nd4j-log-analyzer/target/nd4j-log-analyzer-1.0-SNAPSHOT.jar
|
||||
```
|
||||
|
||||
Make sure to replace the paths with the appropriate locations on your system.
|
||||
|
||||
## Configuration
|
||||
|
||||
The agent uses two main configuration options:
|
||||
|
||||
1. `sourceCodeIndexerPath`: The path to the DeepLearning4J codebase you want to index.
|
||||
2. `javaagent`: The path to the compiled ND4J Log Analyzer JAR file.
|
||||
|
||||
## Database Structure
|
||||
|
||||
The H2 database is automatically created and managed by the agent. It contains two main tables:
|
||||
|
||||
1. `OpLogEvent`: Stores information about ND4J operation executions.
|
||||
- Columns: id, opName, inputs, outputs, stackTrace, sourceCodeLine
|
||||
|
||||
2. `SourceCodeLine`: Stores indexed source code information (created only if `sourceCodeIndexerPath` is provided).
|
||||
- Columns: id, packageName, className, lineNumber, line, fileName, lastUpdated
|
||||
|
||||
## Data Storage
|
||||
|
||||
- Operation logs are stored in the `OpLogEvent` table.
|
||||
- Each operation execution is recorded with its name, inputs, outputs, stack trace, and corresponding source code line.
|
||||
- Inputs and outputs are stored as arrays of strings.
|
||||
- The source code indexer (if enabled) stores relevant code lines in the `SourceCodeLine` table.
|
||||
|
||||
## StackTraceCodeFinder Utility
|
||||
|
||||
The ND4J Log Analyzer includes a StackTraceCodeFinder utility that helps locate the relevant source code lines for recorded operations. Key features include:
|
||||
|
||||
- Resolves the source file path for a given fully qualified class name
|
||||
- Retrieves the specific line of code from a stack trace element
|
||||
- Caches file paths for improved performance
|
||||
- Skips certain packages to focus on relevant code (configurable skip patterns)
|
||||
- Searches for source roots within the specified directory
|
||||
|
||||
### Usage of StackTraceCodeFinder
|
||||
|
||||
```java
|
||||
String rootDirectory = "/path/to/your/deeplearning4j/codebase";
|
||||
StackTraceElement[] stackTrace = // ... obtain stack trace
|
||||
String sourceCodeLine = StackTraceCodeFinder.getFirstLineOfCode(rootDirectory, stackTrace);
|
||||
```
|
||||
|
||||
This utility is used internally by the Log Analyzer to associate recorded operations with their corresponding source code lines.
|
||||
|
||||
## JsonComparisonReport Utility
|
||||
|
||||
The JsonComparisonReport is a powerful tool for comparing operation logs between different runs or versions of your DeepLearning4J application. It helps identify differences in ND4J operations, which is crucial for detecting regressions or unexpected changes in behavior.
|
||||
|
||||
Key features of the JsonComparisonReport include:
|
||||
|
||||
- Compares operation logs stored in JSON format from two different directories
|
||||
- Supports multiple epsilon values for floating-point comparisons
|
||||
- Generates detailed reports of differences found between operation logs
|
||||
- Identifies the earliest difference in the execution flow
|
||||
- Filters differences based on a specified epsilon threshold
|
||||
|
||||
### Usage of JsonComparisonReport
|
||||
|
||||
To use the JsonComparisonReport, run it as a standalone Java application:
|
||||
|
||||
```
|
||||
java org.nd4j.interceptor.data.JsonComparisonReport <directory1> <directory2>
|
||||
```
|
||||
|
||||
Where:
|
||||
- `<directory1>` is the path to the first set of JSON log files
|
||||
- `<directory2>` is the path to the second set of JSON log files to compare against
|
||||
|
||||
The tool will generate two types of reports for each epsilon value defined in `InterceptorEnvironment.EPSILONS`:
|
||||
|
||||
1. `comparison_report_<epsilon>.json`: A detailed report of all differences found
|
||||
2. `earliest_difference_<epsilon>.json`: Information about the first difference encountered in the execution flow
|
||||
|
||||
These reports can be used to identify and analyze discrepancies between different runs or versions of your DeepLearning4J application.
|
||||
|
||||
## JsonReport Utility
|
||||
|
||||
The JsonReport is a utility tool that generates JSON files for each unique operation name from the recorded ND4J operations. This tool is useful for exporting the collected data in a format that's easy to analyze or compare using other tools.
|
||||
|
||||
Key features of the JsonReport include:
|
||||
|
||||
- Generates a separate JSON file for each unique operation name
|
||||
- Groups operation log events by source code line
|
||||
- Uses a custom ObjectMapper for proper serialization of JSON arrays
|
||||
- Creates a new directory for storing the generated JSON reports
|
||||
|
||||
### Usage of JsonReport
|
||||
|
||||
To use the JsonReport, run it as a standalone Java application:
|
||||
|
||||
```
|
||||
java org.nd4j.interceptor.data.JsonReport <path_to_oplog.db>
|
||||
```
|
||||
|
||||
Where:
|
||||
- `<path_to_oplog.db>` is the path to the H2 database file containing the recorded operations
|
||||
|
||||
The tool will create a new directory called "jsonReports" (or clear it if it already exists) and generate JSON files for each unique operation name found in the database.
|
||||
|
||||
### Output
|
||||
|
||||
For each unique operation name, a JSON file will be created in the "jsonReports" directory. The file name will be `<operation_name>.json`. Each file contains:
|
||||
|
||||
- Grouped operation log events by source code line
|
||||
- Detailed information about each operation execution, including inputs, outputs, and stack traces
|
||||
|
||||
These JSON files can be used for further analysis, comparison between different runs, or as input for other tools like the JsonComparisonReport.
|
||||
|
||||
## Workflow for Analyzing ND4J Operations
|
||||
|
||||
1. Run your DeepLearning4J application with the ND4J Log Analyzer agent to collect operation data.
|
||||
2. Use the JsonReport utility to export the collected data to JSON files:
|
||||
```
|
||||
java org.nd4j.interceptor.data.JsonReport path/to/your/oplog.db
|
||||
```
|
||||
3. If you want to compare two different runs or versions:
|
||||
a. Generate JSON reports for both runs using JsonReport
|
||||
b. Use the JsonComparisonReport to compare the generated JSON files:
|
||||
```
|
||||
java org.nd4j.interceptor.data.JsonComparisonReport path/to/jsonReports1 path/to/jsonReports2
|
||||
```
|
||||
4. Analyze the comparison reports to identify differences or potential regressions in ND4J operations.
|
||||
|
||||
## Analyzing Results
|
||||
|
||||
After running your DeepLearning4J application with the agent, you can query the H2 database to analyze the recorded operations. The `InterceptorPersistence` class provides several methods for data analysis:
|
||||
|
||||
1. Get all unique operation names:
|
||||
```java
|
||||
Set<String> uniqueOpNames = InterceptorPersistence.getUniqueOpNames(filePath);
|
||||
```
|
||||
|
||||
2. Filter operations by name:
|
||||
```java
|
||||
List<OpLogEvent> filteredEvents = InterceptorPersistence.filterByOpName(filePath, opName);
|
||||
```
|
||||
|
||||
3. Group operations by source code line:
|
||||
```java
|
||||
Map<String, List<OpLogEvent>> groupedEvents = InterceptorPersistence.groupedByCodeSortedByEventId(logEvents);
|
||||
```
|
||||
|
||||
You can also use your preferred SQL client or the H2 Console to connect to the database and run custom queries.
|
||||
|
||||
For comparing results between different runs or versions, use the JsonComparisonReport utility as described above.
|
||||
|
||||
For exporting the recorded operations to JSON format for further analysis or comparison, use the JsonReport utility as described in the previous section.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- If the agent fails to attach, ensure that you have the correct paths specified in the VM arguments.
|
||||
- Check the console output for any error messages from the agent.
|
||||
- Verify that you have write permissions in the directory where the H2 database is being created.
|
||||
- If tables are not created properly, you can use the `InterceptorPersistence.listTables()` method to check the existing tables in the database.
|
||||
- If source code lines are not being found, check that the `sourceCodeIndexerPath` is correct and that the StackTraceCodeFinder can access the necessary files.
|
||||
- When using the JsonComparisonReport, ensure that the JSON log files are in the correct format and located in the specified directories.
|
||||
- When using the JsonReport, ensure that the path to the oplog.db file is correct and that you have write permissions in the directory where the JSON files will be created.
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions to the ND4J Log Analyzer are welcome! Please submit pull requests or open issues on the GitHub repository.
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the Apache License 2.0. See the LICENSE file for details.
|
||||
@@ -0,0 +1,161 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>nd4j-log-analyzer</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>nd4j-log-analyzer</name>
|
||||
|
||||
<properties>
|
||||
<nd4j.version>1.0.0-SNAPSHOT</nd4j.version>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<bytebuddy.version>1.14.15</bytebuddy.version>
|
||||
<jackson.version>1.0.0-SNAPSHOT</jackson.version>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<lombok.version>1.18.42</lombok.version>
|
||||
<javaparser.version>3.24.4</javaparser.version>
|
||||
|
||||
</properties>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>3.2.4</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<artifactSet>
|
||||
<includes>
|
||||
<include>org.nd4j:nd4j-log-analyzer</include>
|
||||
<include>com.github.javaparser:*</include>
|
||||
<include>net.bytebuddy:byte-buddy-dep</include>
|
||||
<include>com.tdunning:json</include>
|
||||
<include>net.bytebuddy:byte-buddy-agent</include>
|
||||
<include>com.h2database:h2</include>
|
||||
<include>org.ow2.asm:asm</include>
|
||||
<include>org.ow2.asm:asm-commons</include>
|
||||
<include>>org.ow2.asm:asm-analysis</include>
|
||||
</includes>
|
||||
</artifactSet>
|
||||
<transformers>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
|
||||
<manifestEntries>
|
||||
<Premain-Class>org.nd4j.interceptor.Nd4jInterceptor</Premain-Class>
|
||||
<Can-Retransform-Classes>true</Can-Retransform-Classes>
|
||||
</manifestEntries>
|
||||
</transformer>
|
||||
</transformers>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<version>3.3.0</version>
|
||||
<configuration>
|
||||
<archive>
|
||||
<manifestEntries>
|
||||
<Premain-Class>org.nd4j.interceptor.Nd4jInterceptor</Premain-Class>
|
||||
<Can-Retransform-Classes>true</Can-Retransform-Classes>
|
||||
</manifestEntries>
|
||||
</archive>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<dependencies>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>com.tdunning</groupId>
|
||||
<artifactId>json</artifactId>
|
||||
<version>1.8</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.ow2.asm</groupId>
|
||||
<artifactId>asm-analysis</artifactId>
|
||||
<version>9.6</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.github.javaparser</groupId>
|
||||
<artifactId>javaparser-core-serialization</artifactId>
|
||||
<version>${javaparser.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.github.javaparser</groupId>
|
||||
<artifactId>javaparser-symbol-solver-core</artifactId>
|
||||
<version>${javaparser.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>nd4j-common</artifactId>
|
||||
<version>${nd4j.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>nd4j-native</artifactId>
|
||||
<version>${nd4j.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>deeplearning4j-nn</artifactId>
|
||||
<version>${nd4j.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<version>${lombok.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>jackson</artifactId>
|
||||
<version>${jackson.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.bytebuddy</groupId>
|
||||
<artifactId>byte-buddy-dep</artifactId>
|
||||
<version>${bytebuddy.version}</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>com.tdunning</groupId>
|
||||
<artifactId>json</artifactId>
|
||||
<version>1.8</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>net.bytebuddy</groupId>
|
||||
<artifactId>byte-buddy-agent</artifactId>
|
||||
<version>${bytebuddy.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.h2database</groupId>
|
||||
<artifactId>h2</artifactId>
|
||||
<version>2.2.224</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
|
||||
|
||||
</dependencies>
|
||||
</project>
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.interceptor;
|
||||
|
||||
import org.nd4j.interceptor.data.JSONArraySerializer;
|
||||
import org.nd4j.shade.jackson.databind.ObjectMapper;
|
||||
import org.nd4j.shade.jackson.databind.SerializationFeature;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
public class InterceptorEnvironment {
|
||||
public static final String CURRENT_FILE_PATH = new File("oplog.db").getAbsolutePath();
|
||||
public static final String USER = "nd4j";
|
||||
public static final String PASSWORD = "nd4j";
|
||||
public static final String SOURCE_CODE_INDEXER_PATH_KEY = "sourceCodeIndexerPath";
|
||||
public static final String SOURCE_CODE_INDEXER_PATH = System.getProperty(SOURCE_CODE_INDEXER_PATH_KEY);
|
||||
public static final ObjectMapper mapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT)
|
||||
.registerModule(new JSONArraySerializer.JSONArraySerializerModule());
|
||||
public static final double[] EPSILONS = {1e-3, 1e-6, 1e-12};
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor;
|
||||
|
||||
import net.bytebuddy.agent.builder.AgentBuilder;
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import net.bytebuddy.description.type.TypeDescription;
|
||||
import net.bytebuddy.dynamic.ClassFileLocator;
|
||||
import net.bytebuddy.dynamic.DynamicType;
|
||||
import net.bytebuddy.implementation.MethodDelegation;
|
||||
import net.bytebuddy.matcher.ElementMatchers;
|
||||
import net.bytebuddy.utility.JavaModule;
|
||||
import org.deeplearning4j.nn.api.Layer;
|
||||
import org.deeplearning4j.nn.conf.graph.LayerVertex;
|
||||
import org.deeplearning4j.nn.graph.ComputationGraph;
|
||||
import org.deeplearning4j.nn.graph.vertex.GraphVertex;
|
||||
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
|
||||
import org.nd4j.interceptor.advice.MultiLayerNetworkBackwardAdvice;
|
||||
import org.nd4j.interceptor.advice.MultiLayerNetworkForwardAdvice;
|
||||
import org.nd4j.interceptor.data.InterceptorPersistence;
|
||||
import org.nd4j.interceptor.transformers.*;
|
||||
import org.nd4j.interceptor.util.InterceptorUtils;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.api.ops.executioner.OpExecutioner;
|
||||
|
||||
import java.io.File;
|
||||
import java.lang.instrument.Instrumentation;
|
||||
import java.security.ProtectionDomain;
|
||||
|
||||
import static net.bytebuddy.matcher.ElementMatchers.none;
|
||||
|
||||
public class Nd4jInterceptor {
|
||||
|
||||
|
||||
public static void premain(String agentArgs, Instrumentation inst) {
|
||||
AgentBuilder agentBuilder = new AgentBuilder.Default()
|
||||
.ignore(none())
|
||||
.with(AgentBuilder.RedefinitionStrategy.RETRANSFORMATION)
|
||||
.type(ElementMatchers.nameContains("MultiLayerNetwork"))
|
||||
.transform(new MultiLayerNetworkTransformer());
|
||||
|
||||
|
||||
agentBuilder.installOn(inst);
|
||||
|
||||
|
||||
AgentBuilder agentBuilder6 = new AgentBuilder.Default()
|
||||
.ignore(none())
|
||||
.with(AgentBuilder.RedefinitionStrategy.RETRANSFORMATION)
|
||||
.type(ElementMatchers.nameContains("ComputationGraph"))
|
||||
.transform(new ComputationGraphTransformer());
|
||||
|
||||
|
||||
agentBuilder6.installOn(inst);
|
||||
|
||||
AgentBuilder agentBuilder2 = new AgentBuilder.Default()
|
||||
.ignore(none())
|
||||
.with(AgentBuilder.RedefinitionStrategy.RETRANSFORMATION)
|
||||
.type(ElementMatchers.isSubTypeOf(Layer.class))
|
||||
.transform(new LayerTransformer());
|
||||
|
||||
agentBuilder2.installOn(inst);
|
||||
|
||||
AgentBuilder agentBuilder3 = new AgentBuilder.Default()
|
||||
.ignore(none())
|
||||
.with(AgentBuilder.RedefinitionStrategy.RETRANSFORMATION)
|
||||
.type(ElementMatchers.isSubTypeOf(GraphVertex.class))
|
||||
.transform(new ComputationGraphVertexTransformer());
|
||||
|
||||
agentBuilder3.installOn(inst);
|
||||
|
||||
|
||||
|
||||
AgentBuilder agentBuilder4 = new AgentBuilder.Default()
|
||||
.ignore(none())
|
||||
.with(AgentBuilder.RedefinitionStrategy.RETRANSFORMATION)
|
||||
.type(ElementMatchers.isSubTypeOf(OpExecutioner.class))
|
||||
.transform(new OpExecutionerTransformer());
|
||||
|
||||
agentBuilder4.installOn(inst);
|
||||
|
||||
|
||||
AgentBuilder agentBuilder5 = new AgentBuilder.Default()
|
||||
.with(AgentBuilder.RedefinitionStrategy.RETRANSFORMATION)
|
||||
.type(ElementMatchers.isSubTypeOf(INDArray.class).or(ElementMatchers.named("BaseNDArray")))
|
||||
.transform(new INDArrayTransformer());
|
||||
agentBuilder5.installOn(inst);
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package org.nd4j.interceptor.advice;
|
||||
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import org.deeplearning4j.nn.gradient.Gradient;
|
||||
import org.nd4j.common.primitives.AtomicBoolean;
|
||||
import org.nd4j.common.primitives.Pair;
|
||||
import org.nd4j.interceptor.data.InterceptorPersistence;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
|
||||
public class ComputationGraphBackwardAdvice {
|
||||
public static final ThreadLocal<AtomicBoolean> calcBackpropScope = ThreadLocal.withInitial(() -> new AtomicBoolean(false));
|
||||
|
||||
public static boolean isCalcBackpropScope() {
|
||||
return calcBackpropScope.get().get();
|
||||
}
|
||||
|
||||
|
||||
@Advice.OnMethodEnter
|
||||
public static void enter(@Advice.This Object thisObject,
|
||||
@Advice.Origin("#m") String detailedOrigin) {
|
||||
calcBackpropScope.get().set(true);
|
||||
|
||||
}
|
||||
|
||||
@Advice.OnMethodExit
|
||||
public static void exit(@Advice.This Object thisObject,
|
||||
@Advice.Origin("#m") String detailedOrigin) {
|
||||
InterceptorPersistence.finishCurrentBackwardPass();
|
||||
calcBackpropScope.get().set(false);
|
||||
|
||||
}
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.advice;
|
||||
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import org.nd4j.common.primitives.AtomicBoolean;
|
||||
import org.nd4j.interceptor.data.InterceptorPersistence;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
|
||||
import static org.nd4j.interceptor.data.InterceptorPersistence.finishCurrentForwardPass;
|
||||
|
||||
public class ComputationGraphForwardAdvice {
|
||||
public static final ThreadLocal<AtomicBoolean> calcForwardScope = ThreadLocal.withInitial(() -> new AtomicBoolean(false));
|
||||
|
||||
public static boolean isCalcForwardScope() {
|
||||
return calcForwardScope.get().get();
|
||||
}
|
||||
|
||||
@Advice.OnMethodEnter
|
||||
public static void enter(@Advice.Origin("#m") String methodName) {
|
||||
calcForwardScope.get().set(true);
|
||||
}
|
||||
|
||||
@Advice.OnMethodExit
|
||||
public static void exit(@Advice.Origin("#m") String methodName) {
|
||||
calcForwardScope.get().set(false);
|
||||
finishCurrentForwardPass();
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.advice;
|
||||
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import org.deeplearning4j.nn.gradient.Gradient;
|
||||
import org.deeplearning4j.nn.graph.ComputationGraph;
|
||||
import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr;
|
||||
import org.nd4j.common.primitives.Pair;
|
||||
import org.nd4j.interceptor.data.InterceptorPersistence;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
|
||||
public class ComputationGraphVertexDoBackwardAdvice {
|
||||
|
||||
|
||||
@Advice.OnMethodExit
|
||||
public static void exit(@Advice.This ComputationGraph graph,
|
||||
@Advice.Return Pair<Gradient, INDArray[]> result,
|
||||
@Advice.Origin("#t") String className) {
|
||||
|
||||
InterceptorPersistence.addToBackwardPass(result.getSecond());
|
||||
}
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.advice;
|
||||
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import org.deeplearning4j.nn.graph.ComputationGraph;
|
||||
import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr;
|
||||
import org.nd4j.interceptor.data.InterceptorPersistence;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
|
||||
|
||||
|
||||
public class ComputationGraphVertexDoForwardAdvice {
|
||||
|
||||
|
||||
@Advice.OnMethodExit
|
||||
public static void exit( @Advice.Return INDArray[] output) {
|
||||
InterceptorPersistence.addToForwardPass(output);
|
||||
}
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.advice;
|
||||
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import org.nd4j.interceptor.util.InterceptorUtils;
|
||||
import org.nd4j.linalg.api.ops.CustomOp;
|
||||
|
||||
public class CustomOpAdvice {
|
||||
@Advice.OnMethodExit
|
||||
public static void exit(@Advice.AllArguments Object[] args) {
|
||||
if (args != null && args.length > 0) {
|
||||
Object opOrCustomOp = args[0];
|
||||
CustomOp customOp = (CustomOp) opOrCustomOp;
|
||||
InterceptorUtils.logCustomOpExecution(customOp);
|
||||
}
|
||||
}
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.advice;
|
||||
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
|
||||
|
||||
public class INDArrayCreationAdvice {
|
||||
@Advice.OnMethodExit
|
||||
public static void exit(@Advice.This INDArray array,
|
||||
@Advice.Origin("#t") String className,
|
||||
@Advice.Origin("#m") String methodName) {
|
||||
NDArrayIndexCounter.increment(className, methodName);
|
||||
}
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.advice;
|
||||
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
|
||||
|
||||
public class INDArrayUpdateAdvice {
|
||||
|
||||
|
||||
@Advice.OnMethodEnter
|
||||
public static void enter(@Advice.This INDArray array,
|
||||
@Advice.Origin("#t") String className,
|
||||
@Advice.Origin("#m") String methodName) {
|
||||
NDArrayIndexCounter.increment(className, methodName);
|
||||
}
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.advice;
|
||||
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import org.deeplearning4j.nn.api.Layer;
|
||||
import org.nd4j.interceptor.data.InterceptorPersistence;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
|
||||
public class LayerActivateWithInputAdvice {
|
||||
|
||||
@Advice.OnMethodExit
|
||||
public static void exit(@Advice.Return INDArray output) {
|
||||
InterceptorPersistence.addToForwardPass(output);
|
||||
|
||||
}
|
||||
}
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.advice;
|
||||
|
||||
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import org.deeplearning4j.nn.api.Layer;
|
||||
import org.deeplearning4j.nn.gradient.Gradient;
|
||||
import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr;
|
||||
import org.nd4j.common.primitives.Pair;
|
||||
import org.nd4j.interceptor.data.InterceptorPersistence;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class LayerBackpropGradientAdvice {
|
||||
@Advice.OnMethodEnter
|
||||
public static void enter( @Advice.Argument(0) INDArray epsilon) {
|
||||
if(epsilon != null) {
|
||||
InterceptorPersistence.addToBackwardPass(epsilon);
|
||||
}
|
||||
}
|
||||
|
||||
@Advice.OnMethodExit
|
||||
public static void exit(@Advice.Return Pair<Gradient, INDArray> result) {
|
||||
if (result != null) {
|
||||
Gradient gradient = result.getFirst();
|
||||
if (gradient != null) {
|
||||
for (Map.Entry<String, INDArray> entry : gradient.gradientForVariable().entrySet()) {
|
||||
INDArray gradientArray = entry.getValue();
|
||||
if (gradientArray != null) {
|
||||
InterceptorPersistence.addToBackwardPass(entry.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.advice;
|
||||
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import org.nd4j.common.primitives.AtomicBoolean;
|
||||
import org.nd4j.interceptor.data.InterceptorPersistence;
|
||||
|
||||
public class MultiLayerNetworkBackwardAdvice {
|
||||
|
||||
|
||||
public static final ThreadLocal<AtomicBoolean> calcBackpropScope = ThreadLocal.withInitial(() -> new AtomicBoolean(false));
|
||||
|
||||
public static boolean isCalcBackpropScope() {
|
||||
return calcBackpropScope.get().get();
|
||||
}
|
||||
|
||||
|
||||
@Advice.OnMethodEnter
|
||||
public static void enter(@Advice.This Object thisObject,
|
||||
@Advice.Origin("#m") String detailedOrigin) {
|
||||
calcBackpropScope.get().set(true);
|
||||
|
||||
}
|
||||
|
||||
@Advice.OnMethodExit
|
||||
public static void exit(@Advice.This Object thisObject,
|
||||
@Advice.Origin("#m") String detailedOrigin) {
|
||||
InterceptorPersistence.finishCurrentBackwardPass();
|
||||
calcBackpropScope.get().set(false);
|
||||
}
|
||||
}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.advice;
|
||||
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import org.nd4j.common.primitives.AtomicBoolean;
|
||||
import org.nd4j.interceptor.data.InterceptorPersistence;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
|
||||
import static org.nd4j.interceptor.data.InterceptorPersistence.finishCurrentForwardPass;
|
||||
|
||||
public class MultiLayerNetworkForwardAdvice {
|
||||
public static final ThreadLocal<AtomicBoolean> calcForwardScope = ThreadLocal.withInitial(() -> new AtomicBoolean(false));
|
||||
|
||||
public static boolean isCalcForwardScope() {
|
||||
return calcForwardScope.get().get();
|
||||
}
|
||||
|
||||
@Advice.OnMethodEnter
|
||||
public static void enter(@Advice.Origin("#m") String methodName) {
|
||||
calcForwardScope.get().set(true);
|
||||
}
|
||||
|
||||
@Advice.OnMethodExit
|
||||
public static void exit(@Advice.Origin("#m") String methodName) {
|
||||
finishCurrentForwardPass();
|
||||
calcForwardScope.get().set(false);
|
||||
|
||||
}
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.advice;
|
||||
|
||||
import org.nd4j.common.primitives.CounterMap;
|
||||
|
||||
public class NDArrayIndexCounter {
|
||||
|
||||
private static CounterMap<String,String> counterMap = new CounterMap<>();
|
||||
|
||||
|
||||
public static int getCount(String className,String methodName) {
|
||||
return (int) counterMap.getCount(className,methodName);
|
||||
}
|
||||
public static void increment(String className,String methodName) {
|
||||
counterMap.incrementCount(className,methodName,1.0);
|
||||
}
|
||||
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
package org.nd4j.interceptor.advice;
|
||||
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import org.nd4j.interceptor.util.InterceptorUtils;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.api.ops.Op;
|
||||
|
||||
public class OpExecutionerAdvice {
|
||||
@Advice.OnMethodExit
|
||||
public static void exit(@Advice.AllArguments Object[] args) {
|
||||
if (args != null && args.length > 0) {
|
||||
Object opOrCustomOp = args[0];
|
||||
if (opOrCustomOp instanceof Op) {
|
||||
Op op = (Op) opOrCustomOp;
|
||||
InterceptorUtils.logOpExecution(op);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void error(@Advice.Thrown Throwable t) {
|
||||
t.printStackTrace();
|
||||
}
|
||||
}
|
||||
+280
@@ -0,0 +1,280 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.data;
|
||||
|
||||
import org.nd4j.interceptor.InterceptorEnvironment;
|
||||
import org.nd4j.interceptor.parser.SourceCodeIndexer;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
|
||||
import java.io.File;
|
||||
import java.sql.*;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class InterceptorPersistence {
|
||||
|
||||
public static final Map<Long, StackTraceElement[]> arrCreationTraces = new ConcurrentHashMap<>();
|
||||
|
||||
|
||||
public static StackTraceElement[] getCreationTrace(long id) {
|
||||
return arrCreationTraces.get(id);
|
||||
}
|
||||
|
||||
public static StackTraceElement[] getCreationTrace(INDArray arr) {
|
||||
return getCreationTrace(arr.getId());
|
||||
}
|
||||
|
||||
public static void finishCurrentBackwardPass() {
|
||||
|
||||
}
|
||||
|
||||
public static void finishCurrentForwardPass() {
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public static void addOpLog(OpLogEvent logEvent) {
|
||||
insertIntoDatabase(InterceptorEnvironment.CURRENT_FILE_PATH, logEvent);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public static void addToBackwardPass(INDArray...arrs) {
|
||||
|
||||
}
|
||||
|
||||
public static void addToForwardPass(INDArray...arrs) {
|
||||
|
||||
}
|
||||
|
||||
public static void addToForwardPass(INDArray arr) {
|
||||
addToForwardPass(new INDArray[]{arr});
|
||||
}
|
||||
|
||||
public static void addToBackwardPass(INDArray arr) {
|
||||
addToBackwardPass(new INDArray[]{arr});
|
||||
}
|
||||
|
||||
public static void bootstrapDatabase(String filePath) throws SQLException {
|
||||
System.out.println("Bootstrapping database");
|
||||
String jdbcUrl = "jdbc:h2:file:" + filePath;
|
||||
createDbUser(filePath);
|
||||
|
||||
try(Connection conn = DriverManager.getConnection(jdbcUrl, InterceptorEnvironment.USER, InterceptorEnvironment.PASSWORD)) {
|
||||
createOpLogEventTable(conn);
|
||||
conn.commit();
|
||||
}
|
||||
}
|
||||
|
||||
public static void createDbUser(String filePath) throws SQLException {
|
||||
String jdbcUrl = "jdbc:h2:file:" + filePath;
|
||||
Connection conn = DriverManager.getConnection(jdbcUrl, "SA", "");
|
||||
try {
|
||||
Statement stmt = conn.createStatement();
|
||||
//user sql: create user if not exists scott password 'tiger' admin;
|
||||
stmt.execute("create user if not exists nd4j password 'nd4j' admin");
|
||||
} finally {
|
||||
conn.commit();
|
||||
conn.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static void createOpLogEventTable(Connection conn) throws SQLException {
|
||||
try (Statement stmt = conn.createStatement()) {
|
||||
// Drop OpLogEvent table if it exists
|
||||
String dropTableSql = "DROP TABLE IF EXISTS OpLogEvent";
|
||||
stmt.execute(dropTableSql);
|
||||
|
||||
// Create new OpLogEvent table
|
||||
String createTableSql = "CREATE TABLE OpLogEvent ("
|
||||
+ "id bigint auto_increment, "
|
||||
+ "opName VARCHAR(255), "
|
||||
+ "inputs LONGVARCHAR ARRAY, " // inputs are stored as an array
|
||||
+ "outputs LONGVARCHAR ARRAY, " // outputs are stored as an array
|
||||
+ "stackTrace LONGVARCHAR," // stackTrace is stored as a string
|
||||
+ "sourceCodeLine LONGVARCHAR" // stackTrace is stored as a string
|
||||
+ ")";
|
||||
stmt.execute(createTableSql);
|
||||
System.out.println("Created OpLogEvent table.");
|
||||
}
|
||||
}
|
||||
|
||||
public static void createSourceCodeLineTable(String filePath, Connection conn) throws SQLException {
|
||||
try (Statement stmt = conn.createStatement()) {
|
||||
// Check if the SOURCE_CODE_INDEXER_PATH system property is defined
|
||||
if (InterceptorEnvironment.SOURCE_CODE_INDEXER_PATH != null) {
|
||||
System.out.println("Creating SourceCodeLine table");
|
||||
// Create new SourceCodeLine table
|
||||
String createTableQuery = "CREATE TABLE IF NOT EXISTS SourceCodeLine (" +
|
||||
"id BIGINT AUTO_INCREMENT PRIMARY KEY," +
|
||||
"packageName LONGVARCHAR," +
|
||||
"className LONGVARCHAR," +
|
||||
"lineNumber INT," +
|
||||
"line LONGVARCHAR," +
|
||||
"fileName LONGVARCHAR," +
|
||||
"lastUpdated TIMESTAMP DEFAULT CURRENT_TIMESTAMP" +
|
||||
")";
|
||||
stmt.execute(createTableQuery);
|
||||
System.out.println("Created SourceCodeLine table.");
|
||||
|
||||
// Create a SourceCodeIndexer and index the source code
|
||||
SourceCodeIndexer sourceCodeIndexer = new SourceCodeIndexer(new File(InterceptorEnvironment.SOURCE_CODE_INDEXER_PATH),filePath);
|
||||
|
||||
// Persist the source code index to the OpLog
|
||||
sourceCodeIndexer.persistToOpLog(filePath);
|
||||
} else {
|
||||
System.out.println("SOURCE_CODE_INDEXER_PATH system property not defined. Skipping SourceCodeLine table creation.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static List<String> listTables(String filePath) {
|
||||
List<String> tables = new ArrayList<>();
|
||||
try {
|
||||
String jdbcUrl = "jdbc:h2:file:" + filePath;
|
||||
Connection conn = DriverManager.getConnection(jdbcUrl, InterceptorEnvironment.USER, InterceptorEnvironment.PASSWORD);
|
||||
DatabaseMetaData md = conn.getMetaData();
|
||||
ResultSet rs = md.getTables(null, null, "%", null);
|
||||
while (rs.next()) {
|
||||
tables.add(rs.getString(3));
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException("Failed to list tables", e);
|
||||
}
|
||||
return tables;
|
||||
}
|
||||
|
||||
public static void insertIntoDatabase(String filePath, OpLogEvent logEvent) {
|
||||
String jdbcUrl = "jdbc:h2:file:" + filePath;
|
||||
|
||||
try (Connection conn = DriverManager.getConnection(jdbcUrl, InterceptorEnvironment.USER, InterceptorEnvironment.PASSWORD);
|
||||
PreparedStatement stmt = conn.prepareStatement("INSERT INTO OpLogEvent (opName, inputs, outputs, stackTrace,sourceCodeLine) VALUES (?, ?, ?, ?,?)")) {
|
||||
|
||||
if(logEvent.firstNonExecutionCodeLine == null) {
|
||||
throw new IllegalArgumentException("Source code line should not be null.");
|
||||
}
|
||||
stmt.setString(1, logEvent.getOpName());
|
||||
stmt.setArray(2, conn.createArrayOf("VARCHAR", convertMapToArray(logEvent.getInputs())));
|
||||
stmt.setArray(3, conn.createArrayOf("VARCHAR", convertMapToArray(logEvent.getOutputs())));
|
||||
stmt.setString(4, logEvent.getStackTrace());
|
||||
stmt.setString(5,logEvent.firstNonExecutionCodeLine.trim());
|
||||
stmt.executeUpdate();
|
||||
} catch(Exception e) {
|
||||
throw new RuntimeException("Failed to insert OpLogEvent into database", e);
|
||||
}
|
||||
}
|
||||
|
||||
public static String[] convertMapToArray(Map<Integer, String> map) {
|
||||
// Create a new array with the same size as the map
|
||||
String[] array = new String[map.size()];
|
||||
|
||||
// Iterate over the map entries
|
||||
for (Map.Entry<Integer, String> entry : map.entrySet()) {
|
||||
// Get the key (integer) of the current entry
|
||||
int key = entry.getKey();
|
||||
|
||||
// Use the key as the index in the array and assign the corresponding value
|
||||
array[key] = entry.getValue();
|
||||
}
|
||||
|
||||
return array;
|
||||
}
|
||||
|
||||
public static List<String> listTables() {
|
||||
List<String> tables = new ArrayList<>();
|
||||
try {
|
||||
String jdbcUrl = "jdbc:h2:file:" + InterceptorEnvironment.CURRENT_FILE_PATH;
|
||||
Connection conn = DriverManager.getConnection(jdbcUrl, InterceptorEnvironment.USER, InterceptorEnvironment.PASSWORD);
|
||||
DatabaseMetaData md = conn.getMetaData();
|
||||
ResultSet rs = md.getTables(null, null, "%", null);
|
||||
while (rs.next()) {
|
||||
tables.add(rs.getString(3));
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException("Failed to list tables", e);
|
||||
}
|
||||
return tables;
|
||||
}
|
||||
|
||||
|
||||
public static Map<Integer,String> convertResult(Object input) {
|
||||
Object[] inputArr = (Object[]) input;
|
||||
Map<Integer,String> ret = new LinkedHashMap<>();
|
||||
for (int i = 0; i < inputArr.length; i++) {
|
||||
ret.put(i,inputArr[i].toString());
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
public static Map<String,List<OpLogEvent>> groupedByCodeSortedByEventId(List<OpLogEvent> logEvents) {
|
||||
return logEvents.stream().collect(Collectors.groupingBy(OpLogEvent::getFirstNonExecutionCodeLine, Collectors.toList()));
|
||||
}
|
||||
|
||||
public static List<OpLogEvent> filterByOpName(String filePath, String opName) throws SQLException {
|
||||
List<OpLogEvent> filteredEvents = new ArrayList<>();
|
||||
|
||||
String jdbcUrl = "jdbc:h2:file:" + filePath;
|
||||
|
||||
try (Connection conn = DriverManager.getConnection(jdbcUrl, InterceptorEnvironment.USER, InterceptorEnvironment.PASSWORD);
|
||||
PreparedStatement stmt = conn.prepareStatement("SELECT * FROM OpLogEvent WHERE opName = ?")) {
|
||||
|
||||
stmt.setString(1, opName);
|
||||
|
||||
try (ResultSet rs = stmt.executeQuery()) {
|
||||
while (rs.next()) {
|
||||
OpLogEvent event = OpLogEvent.builder()
|
||||
.firstNonExecutionCodeLine(rs.getString("sourceCodeLine"))
|
||||
.eventId(rs.getLong("id"))
|
||||
.opName(rs.getString("opName"))
|
||||
.inputs(convertResult((rs.getArray("inputs").getArray())))
|
||||
.outputs(convertResult(rs.getArray("outputs").getArray()))
|
||||
.stackTrace(rs.getString("stackTrace"))
|
||||
.build();
|
||||
filteredEvents.add(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return filteredEvents;
|
||||
}
|
||||
|
||||
public static Set<String> getUniqueOpNames(String filePath) throws SQLException {
|
||||
Set<String> uniqueOpNames = new HashSet<>();
|
||||
String jdbcUrl = "jdbc:h2:file:" + filePath;
|
||||
|
||||
try (Connection conn = DriverManager.getConnection(jdbcUrl, InterceptorEnvironment.USER, InterceptorEnvironment.PASSWORD);
|
||||
Statement stmt = conn.createStatement();
|
||||
ResultSet rs = stmt.executeQuery("SELECT DISTINCT opName FROM OPLOGEVENT")) {
|
||||
|
||||
while (rs.next()) {
|
||||
uniqueOpNames.add(rs.getString("opName"));
|
||||
}
|
||||
}
|
||||
|
||||
return uniqueOpNames;
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.interceptor.data;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.nd4j.shade.jackson.core.JsonGenerator;
|
||||
import org.nd4j.shade.jackson.databind.JsonSerializer;
|
||||
import org.nd4j.shade.jackson.databind.SerializerProvider;
|
||||
import org.nd4j.shade.jackson.databind.module.SimpleModule;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class JSONArraySerializer extends JsonSerializer<JSONArray> {
|
||||
|
||||
@Override
|
||||
public void serialize(JSONArray value, JsonGenerator gen, SerializerProvider serializers) throws IOException {
|
||||
gen.writeStartArray();
|
||||
for (int i = 0; i < value.length(); i++) {
|
||||
Object item = value.opt(i);
|
||||
if (item == null) {
|
||||
gen.writeNull();
|
||||
} else if (item instanceof Boolean) {
|
||||
gen.writeBoolean((Boolean) item);
|
||||
} else if (item instanceof Integer) {
|
||||
gen.writeNumber((Integer) item);
|
||||
} else if (item instanceof Long) {
|
||||
gen.writeNumber((Long) item);
|
||||
} else if (item instanceof Double) {
|
||||
gen.writeNumber((Double) item);
|
||||
} else if (item instanceof String) {
|
||||
gen.writeString((String) item);
|
||||
} else if (item instanceof JSONArray) {
|
||||
serialize((JSONArray) item, gen, serializers);
|
||||
} else {
|
||||
gen.writeObject(item.toString());
|
||||
}
|
||||
}
|
||||
gen.writeEndArray();
|
||||
}
|
||||
|
||||
public static class JSONArraySerializerModule extends SimpleModule {
|
||||
public JSONArraySerializerModule() {
|
||||
addSerializer(JSONArray.class, new JSONArraySerializer());
|
||||
}
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.interceptor.data;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class JSONComparisonResult {
|
||||
@Builder.Default
|
||||
private int index = -1;
|
||||
@Builder.Default
|
||||
private boolean same = true;
|
||||
private double firstValue;
|
||||
private double secondValue;
|
||||
|
||||
public static JSONComparisonResult noDifference() {
|
||||
return JSONComparisonResult.builder().same(false).build();
|
||||
}
|
||||
}
|
||||
+403
@@ -0,0 +1,403 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.interceptor.data;
|
||||
import org.json.JSONTokener;
|
||||
import org.nd4j.interceptor.InterceptorEnvironment;
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileReader;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.*;
|
||||
|
||||
|
||||
public class JsonComparisonReport {
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
if (args.length != 2) {
|
||||
System.out.println("Usage: java JsonComparisonReport <directory1> <directory2>");
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
String directory1 = args[0];
|
||||
String directory2 = args[1];
|
||||
for(double epsilon : InterceptorEnvironment.EPSILONS) {
|
||||
Map<String,OpDifference> differences = compareDirectories(directory1, directory2,epsilon);
|
||||
generateReport(differences,epsilon);
|
||||
}
|
||||
|
||||
List<OpLogEvent> orderedEvents1 = orderedEvents(new File(directory1));
|
||||
List<OpLogEvent> orderedEvents2 = orderedEvents(new File(directory2));
|
||||
try {
|
||||
InterceptorEnvironment.mapper.writeValue(new File("first_in_order.json"), orderedEvents1);
|
||||
InterceptorEnvironment.mapper.writeValue(new File("second_in_order.json"), orderedEvents2);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
private static Map<String,OpDifference> compareDirectories(String directory1, String directory2,double epsilon) {
|
||||
Map<String,OpDifference> differences = new HashMap<>();
|
||||
File dir1 = new File(directory1);
|
||||
File dir2 = new File(directory2);
|
||||
|
||||
File[] files1 = dir1.listFiles((dir, name) -> name.endsWith(".json"));
|
||||
File[] files2 = dir2.listFiles((dir, name) -> name.endsWith(".json"));
|
||||
|
||||
if (files1 != null && files2 != null) {
|
||||
for (File file1 : files1) {
|
||||
if(file1.getName().contains("div_scalar")) {
|
||||
continue;
|
||||
}
|
||||
String fileName = file1.getName();
|
||||
File file2 = new File(dir2, fileName);
|
||||
|
||||
if (file2.exists()) {
|
||||
try {
|
||||
System.out.println("Processing files: " + file1.getName() + " and " + file2.getName());
|
||||
JSONObject jsonObject = new JSONObject(new JSONTokener(new FileReader(file1)));
|
||||
JSONObject jsonObject2 = new JSONObject(new JSONTokener(new FileReader(file2)));
|
||||
|
||||
SourceCodeOpEvent eventsGrouped = convertJsonToSourceCodeOpEvent(jsonObject);
|
||||
SourceCodeOpEvent eventsGrouped2 = convertJsonToSourceCodeOpEvent(jsonObject2);
|
||||
Map<String, OpDifference> opLogDifferences = compareOpLogArrays(eventsGrouped.getOpLogEvents(), eventsGrouped2.getOpLogEvents(),epsilon);
|
||||
differences.putAll(opLogDifferences);
|
||||
} catch (IOException | JSONException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return differences;
|
||||
}
|
||||
|
||||
private static SourceCodeOpEvent convertJsonToSourceCodeOpEvent(JSONObject jsonObject) {
|
||||
Map<String, List<OpLogEvent>> opLogEvents = new HashMap<>();
|
||||
jsonObject = jsonObject.getJSONObject("opLogEvents");
|
||||
// Iterate over the keys in the JSON object
|
||||
for (String key : jsonObject.keySet()) {
|
||||
// Get the JSONArray corresponding to the key
|
||||
JSONArray jsonArray = jsonObject.getJSONArray(key);
|
||||
List<OpLogEvent> opLogEventList = new ArrayList<>();
|
||||
|
||||
// Iterate over the elements in the JSONArray
|
||||
for (int i = 0; i < jsonArray.length(); i++) {
|
||||
// Get the JSONObject representing an OpLogEvent
|
||||
JSONObject opLogEventJson = jsonArray.getJSONObject(i);
|
||||
|
||||
// Convert the JSONObject to an OpLogEvent
|
||||
OpLogEvent opLogEvent = convertToOpLogEvent(opLogEventJson);
|
||||
|
||||
// Add the OpLogEvent to the list
|
||||
opLogEventList.add(opLogEvent);
|
||||
}
|
||||
|
||||
// Add the list of OpLogEvents to the map with the corresponding key
|
||||
opLogEvents.put(key, opLogEventList);
|
||||
}
|
||||
|
||||
// Create and return a new SourceCodeOpEvent with the opLogEvents map
|
||||
return SourceCodeOpEvent.builder()
|
||||
.opLogEvents(opLogEvents)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private static List<OpLogEvent> orderedEvents(File directory) {
|
||||
List<OpLogEvent> orderedEvents = new ArrayList<>();
|
||||
File[] files = directory.listFiles((dir, name) -> name.endsWith(".json"));
|
||||
if (files != null) {
|
||||
for (File file : files) {
|
||||
try {
|
||||
JSONObject jsonObject = new JSONObject(new JSONTokener(new FileReader(file)));
|
||||
jsonObject = jsonObject.getJSONObject("opLogEvents");
|
||||
for (String key : jsonObject.keySet()) {
|
||||
JSONArray jsonArray = jsonObject.getJSONArray(key);
|
||||
for (int i = 0; i < jsonArray.length(); i++) {
|
||||
JSONObject opLogEventJson = jsonArray.getJSONObject(i);
|
||||
OpLogEvent opLogEvent = convertToOpLogEvent(opLogEventJson);
|
||||
orderedEvents.add(opLogEvent);
|
||||
}
|
||||
}
|
||||
} catch (IOException | JSONException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Collections.sort(orderedEvents, Comparator.comparingLong(OpLogEvent::getEventId));
|
||||
|
||||
return orderedEvents;
|
||||
}
|
||||
|
||||
private static OpLogEvent convertToOpLogEvent(JSONObject jsonObject) {
|
||||
String opName = jsonObject.getString("opName");
|
||||
JSONObject inputsObject = jsonObject.getJSONObject("inputs");
|
||||
JSONObject outputsObject = jsonObject.getJSONObject("outputs");
|
||||
String stackTrace = jsonObject.getString("stackTrace");
|
||||
|
||||
Map<Integer, String> inputs = decodeInputsOutputs(inputsObject);
|
||||
Map<Integer, String> outputs = decodeInputsOutputs(outputsObject);
|
||||
|
||||
return OpLogEvent.builder()
|
||||
.firstNonExecutionCodeLine(jsonObject.getString("firstNonExecutionCodeLine"))
|
||||
.opName(opName)
|
||||
.inputs(inputs)
|
||||
.outputs(outputs)
|
||||
.eventId(jsonObject.getLong("eventId"))
|
||||
.stackTrace(stackTrace)
|
||||
.build();
|
||||
}
|
||||
|
||||
private static Map<Integer, String> decodeInputsOutputs(JSONObject jsonObject) {
|
||||
Map<Integer, String> result = new HashMap<>();
|
||||
|
||||
for (String key : jsonObject.keySet()) {
|
||||
int index = Integer.parseInt(key);
|
||||
String value = jsonObject.getString(key);
|
||||
result.put(index, value);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static Map<String,OpDifference> compareOpLogArrays(Map<String,List<OpLogEvent>> jsonArray1, Map<String,List<OpLogEvent>> jsonArray2,double epsilon) {
|
||||
Map<String,OpDifference> differences = new HashMap<>();
|
||||
for (String key : jsonArray1.keySet()) {
|
||||
List<OpLogEvent> opLogEvents1 = jsonArray1.get(key);
|
||||
List<OpLogEvent> opLogEvents2 = jsonArray2.get(key);
|
||||
if(opLogEvents1 == null || opLogEvents2 == null)
|
||||
continue;
|
||||
int minEventSize = Math.min(opLogEvents1.size(), opLogEvents2.size());
|
||||
if (opLogEvents2 != null) {
|
||||
for (int i = 0; i < minEventSize; i++) {
|
||||
OpLogEvent opLogEvent1 = opLogEvents1.get(i);
|
||||
OpLogEvent opLogEvent2 = opLogEvents2.get(i);
|
||||
Map<Integer,String> inputs = opLogEvent1.getInputs();
|
||||
Map<Integer,String> outputs = opLogEvent1.getOutputs();
|
||||
|
||||
Map<Integer,String> inputs2 = opLogEvent2.getInputs();
|
||||
Map<Integer,String> outputs2 = opLogEvent2.getOutputs();
|
||||
for(int j = 0; j < inputs.size(); j++) {
|
||||
if(inputs.get(j).contains("assign")) {
|
||||
continue;
|
||||
}
|
||||
JSONArray jsonArray = new JSONArray(inputs.get(j));
|
||||
JSONArray jsonArray3 = new JSONArray(inputs2.get(j));
|
||||
JSONComparisonResult result = compareJSONArraysWithEpsilon(jsonArray, jsonArray3, epsilon);
|
||||
if(!result.isSame()) {
|
||||
OpDifference opDifference = OpDifference.builder()
|
||||
.opLog1(opLogEvent1)
|
||||
.opLog2(opLogEvent2)
|
||||
.differenceType("inputs")
|
||||
.differenceValue1(String.valueOf(result.getFirstValue()))
|
||||
.differenceValue2(String.valueOf(result.getSecondValue()))
|
||||
.opDifference(j)
|
||||
.build();
|
||||
differences.put(key, opDifference);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
for(int j = 0; j < outputs.size(); j++) {
|
||||
if(inputs.get(j).contains("assign")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Object cast = outputs.get(j);
|
||||
if(cast instanceof Number) {
|
||||
cast = new double[] {
|
||||
((Number) cast).doubleValue()
|
||||
};
|
||||
} else if(cast instanceof String) {
|
||||
//if string matches a single double between []
|
||||
|
||||
if(cast.toString().matches("-*\\d+\\.\\d+")) {
|
||||
cast = new JSONArray(new double[] {
|
||||
Double.parseDouble((String) cast)
|
||||
});
|
||||
|
||||
} else {
|
||||
cast = new JSONArray(cast.toString());
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
Object cast2 = outputs2.get(j);
|
||||
if(cast2 instanceof Number) {
|
||||
cast2 = new double[] {
|
||||
((Number) cast2).doubleValue()
|
||||
};
|
||||
} else if(cast2 instanceof String) {
|
||||
//if string matches a single double between []
|
||||
|
||||
if(cast2.toString().matches("-*\\d+\\.\\d+")) {
|
||||
cast2 = new JSONArray(new double[] {
|
||||
Double.parseDouble((String) cast2)
|
||||
});
|
||||
|
||||
} else {
|
||||
cast2 = new JSONArray(cast2.toString());
|
||||
}
|
||||
}
|
||||
|
||||
JSONArray casted1 = (JSONArray) cast;
|
||||
JSONArray casted2 = (JSONArray) cast2;
|
||||
|
||||
JSONComparisonResult result = compareJSONArraysWithEpsilon(casted1, casted2, epsilon);
|
||||
if(!result.isSame()) {
|
||||
OpDifference opDifference = OpDifference.builder()
|
||||
.opLog1(opLogEvent1)
|
||||
.opLog2(opLogEvent2)
|
||||
.differenceType("outputs")
|
||||
.differenceValue1(String.valueOf(result.getFirstValue()))
|
||||
.differenceValue2(String.valueOf(result.getSecondValue()))
|
||||
.opDifference(result.getIndex())
|
||||
.build();
|
||||
differences.put(key, opDifference);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
return differences;
|
||||
}
|
||||
|
||||
|
||||
private static void generateReport(Map<String,OpDifference> differences,double epsilon) {
|
||||
String reportFile = "comparison_report_" + epsilon + ".json";
|
||||
String earliestDifferenceFile = "earliest_difference_" + epsilon + ".json";
|
||||
String firstInOrderFile = "first_in_order_" + epsilon + ".json";
|
||||
String secondInOrderFile = "second_in_order_" + epsilon + ".json";
|
||||
Map<String,OpDifference> filteredDifferences = filterDifferencesByEpsilon(differences, epsilon);
|
||||
|
||||
try {
|
||||
InterceptorEnvironment.mapper.writeValue(new File(reportFile), filteredDifferences);
|
||||
InterceptorEnvironment.mapper.writeValue(new File(earliestDifferenceFile), OpDifference.earliestDifference(filteredDifferences));
|
||||
|
||||
System.out.println("Comparison report for epsilon " + epsilon + " saved to: " + reportFile);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static Map<String,OpDifference> filterDifferencesByEpsilon(Map<String,OpDifference> differences, double epsilon) {
|
||||
Map<String,OpDifference> filteredDifferences = new HashMap<>();
|
||||
|
||||
for (Map.Entry<String,OpDifference> difference : differences.entrySet()) {
|
||||
if (isDifferentWithEpsilon(difference.getValue().getOpLog1(), difference.getValue().getOpLog2(), epsilon)) {
|
||||
filteredDifferences.put(difference.getKey(),difference.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
return filteredDifferences;
|
||||
}
|
||||
|
||||
private static Map<String,String> convertIntMap(Map<Integer,String> map) {
|
||||
Map<String,String> newMap = new HashMap<>();
|
||||
for (Map.Entry<Integer,String> entry : map.entrySet()) {
|
||||
newMap.put(entry.getKey().toString(), entry.getValue());
|
||||
}
|
||||
return newMap;
|
||||
}
|
||||
|
||||
|
||||
private static boolean isDifferentWithEpsilon(OpLogEvent left, OpLogEvent right, double epsilon) {
|
||||
JSONObject leftInputs = new JSONObject(convertIntMap(left.getInputs()));
|
||||
JSONObject rightInputs = new JSONObject(convertIntMap(right.getInputs()));
|
||||
JSONObject leftOutputs = new JSONObject(convertIntMap(left.getOutputs()));
|
||||
JSONObject rightOutputs = new JSONObject(convertIntMap(right.getOutputs()));
|
||||
|
||||
return !compareJSONArraysWithEpsilon(leftInputs, rightInputs, epsilon).isSame()
|
||||
|| !compareJSONArraysWithEpsilon(leftOutputs, rightOutputs, epsilon).isSame();
|
||||
}
|
||||
|
||||
|
||||
private static JSONComparisonResult compareJSONArraysWithEpsilon(JSONArray jsonArray1, JSONArray jsonArray2, double epsilon) {
|
||||
if (jsonArray1.length() != jsonArray2.length()) {
|
||||
return JSONComparisonResult.noDifference();
|
||||
}
|
||||
|
||||
for (int i = 0; i < jsonArray1.length(); i++) {
|
||||
Object value1 = jsonArray1.get(i);
|
||||
Object value2 = jsonArray2.get(i);
|
||||
if(value1 instanceof JSONArray) {
|
||||
JSONComparisonResult result = compareJSONArraysWithEpsilon((JSONArray) value1,(JSONArray) value2,epsilon);
|
||||
if(!result.isSame()) {
|
||||
return result;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
if (Math.abs(((Number) value1).doubleValue() - ((Number) value2).doubleValue()) > epsilon) {
|
||||
return JSONComparisonResult.builder()
|
||||
.same(false)
|
||||
.firstValue(((Number) value1).doubleValue())
|
||||
.secondValue(((Number) value2).doubleValue())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
return JSONComparisonResult.noDifference();
|
||||
}
|
||||
|
||||
|
||||
private static JSONComparisonResult compareJSONArraysWithEpsilon(JSONObject jsonArray1, JSONObject jsonArray2, double epsilon) {
|
||||
if (jsonArray1.length() != jsonArray2.length()) {
|
||||
return JSONComparisonResult.noDifference();
|
||||
}
|
||||
|
||||
for (int i = 0; i < jsonArray1.length(); i++) {
|
||||
Object cast1 = jsonArray1.get(String.valueOf(i));
|
||||
if(cast1 instanceof String) {
|
||||
cast1 = new JSONArray(cast1.toString());
|
||||
}
|
||||
|
||||
Object cast2 = jsonArray2.get(String.valueOf(i));
|
||||
if(cast2 instanceof String) {
|
||||
cast2 = new JSONArray(cast2.toString());
|
||||
}
|
||||
JSONArray value1 = (JSONArray) cast1;
|
||||
JSONArray value2 = (JSONArray) cast2;
|
||||
JSONComparisonResult result = compareJSONArraysWithEpsilon(value1,value2,epsilon);
|
||||
if(!result.isSame()) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
return JSONComparisonResult.noDifference();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.interceptor.data;
|
||||
|
||||
import org.nd4j.shade.jackson.databind.ObjectMapper;
|
||||
import org.nd4j.shade.jackson.databind.SerializationFeature;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.io.File;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.nd4j.interceptor.data.InterceptorPersistence.filterByOpName;
|
||||
import static org.nd4j.interceptor.data.InterceptorPersistence.getUniqueOpNames;
|
||||
|
||||
public class JsonReport {
|
||||
|
||||
|
||||
private static final ObjectMapper objectMapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT)
|
||||
.registerModule(new JSONArraySerializer.JSONArraySerializerModule());
|
||||
|
||||
|
||||
public static void main(String...args) throws Exception {
|
||||
if(args.length < 1) {
|
||||
throw new IllegalArgumentException("Please provide the path to the oplog.db file");
|
||||
}
|
||||
final String CURRENT_FILE_PATH = new File(args[0]).getAbsolutePath();
|
||||
|
||||
String directoryPath = "jsonReports";
|
||||
|
||||
try {
|
||||
Path path = Paths.get(directoryPath);
|
||||
|
||||
// Delete directory if it exists
|
||||
if (Files.exists(path)) {
|
||||
Files.walk(path)
|
||||
.map(Path::toFile)
|
||||
.forEach(File::delete);
|
||||
}
|
||||
|
||||
// Create directory
|
||||
Files.createDirectories(path);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Failed to create directory", e);
|
||||
}
|
||||
|
||||
// Generate a JSON file for each unique op name
|
||||
Set<String> uniqueOpNames = getUniqueOpNames(CURRENT_FILE_PATH);
|
||||
for (String opName : uniqueOpNames) {
|
||||
List<OpLogEvent> events = filterByOpName(CURRENT_FILE_PATH, opName);
|
||||
Map<String,List<OpLogEvent>> eventsGrouped = InterceptorPersistence.groupedByCodeSortedByEventId(events);
|
||||
SourceCodeOpEvent sourceCodeOpEvent = SourceCodeOpEvent.builder()
|
||||
.opLogEvents(eventsGrouped)
|
||||
.build();
|
||||
System.out.println("Writing " + events.size() + " events for " + opName);
|
||||
File newFile = new File(directoryPath + "/" + opName + ".json");
|
||||
if(!newFile.exists()) {
|
||||
newFile.createNewFile();
|
||||
}
|
||||
objectMapper.writeValue(newFile, sourceCodeOpEvent);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.data;
|
||||
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.nd4j.shade.jackson.databind.annotation.JsonSerialize;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
public class OpDifference {
|
||||
@JsonSerialize(using = OpLogEventWriteSerializer.class)
|
||||
private OpLogEvent opLog1;
|
||||
@JsonSerialize(using = OpLogEventWriteSerializer.class)
|
||||
private OpLogEvent opLog2;
|
||||
private String differenceType;
|
||||
private int opDifference;
|
||||
private String differenceValue1;
|
||||
private String differenceValue2;
|
||||
public static List<String> skipOps = Arrays.asList(
|
||||
"set_scalar",
|
||||
"old_assign",
|
||||
"assign"
|
||||
);
|
||||
|
||||
public long getEarliestEventTime() {
|
||||
return Math.min(opLog1.getEventId(), opLog2.getEventId());
|
||||
}
|
||||
|
||||
public static OpDifference earliestDifference(Map<String,OpDifference> differenceList) {
|
||||
Map<String,OpDifference> opLog1 = new HashMap<>();
|
||||
for(Map.Entry<String,OpDifference> opDifference : differenceList.entrySet()) {
|
||||
if(skipOps.contains(opDifference.getValue().getOpLog1().opName) || opDifference.getValue().getOpLog1() == null
|
||||
|| opDifference.getValue().getOpLog2() == null || opDifference.getValue().getOpLog2().opName == null || opDifference.getValue().getOpLog1().opName == null)
|
||||
continue;
|
||||
opLog1.put(opDifference.getKey(), opDifference.getValue());
|
||||
}
|
||||
|
||||
|
||||
|
||||
List<OpDifference> opLog1List = new ArrayList<>(opLog1.values());
|
||||
//find the earliest event in oplog1
|
||||
Collections.sort(opLog1List, Comparator.comparingLong(OpDifference::getEarliestEventTime));
|
||||
|
||||
return opLog1List.get(0);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.interceptor.data;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.nd4j.shade.jackson.core.JsonGenerator;
|
||||
import org.nd4j.shade.jackson.databind.JsonSerializer;
|
||||
import org.nd4j.shade.jackson.databind.ObjectMapper;
|
||||
import org.nd4j.shade.jackson.databind.SerializationFeature;
|
||||
import org.nd4j.shade.jackson.databind.SerializerProvider;
|
||||
import org.nd4j.shade.jackson.databind.annotation.JsonSerialize;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
public class OpLogEvent {
|
||||
public String opName;
|
||||
|
||||
@Builder.Default
|
||||
@JsonSerialize(using = InputOutputSerializer.class)
|
||||
public Map<Integer,String> inputs = new LinkedHashMap<>();
|
||||
|
||||
@Builder.Default
|
||||
@JsonSerialize(using = InputOutputSerializer.class)
|
||||
public Map<Integer,String> outputs = new LinkedHashMap();
|
||||
|
||||
@JsonSerialize(using = StackTraceSerializer.class)
|
||||
public String stackTrace;
|
||||
|
||||
public String firstNonExecutionCodeLine;
|
||||
|
||||
|
||||
|
||||
public long eventId;
|
||||
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
OpLogEvent that = (OpLogEvent) o;
|
||||
return Objects.equals(firstNonExecutionCodeLine, that.firstNonExecutionCodeLine) &&
|
||||
Objects.equals(opName, that.opName) &&
|
||||
Objects.equals(inputs, that.inputs) &&
|
||||
Objects.equals(outputs, that.outputs);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(firstNonExecutionCodeLine, opName, inputs, outputs);
|
||||
}
|
||||
|
||||
public static class InputOutputSerializer extends JsonSerializer<Map<Integer, String>> {
|
||||
@Override
|
||||
public void serialize(Map<Integer, String> value, JsonGenerator gen, SerializerProvider serializers) throws IOException {
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
objectMapper.registerModule(new JSONArraySerializer.JSONArraySerializerModule());
|
||||
objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
|
||||
Map<Integer, Object> write = new LinkedHashMap<>();
|
||||
for (Map.Entry<Integer, String> entry : value.entrySet()) {
|
||||
Integer key = entry.getKey();
|
||||
String item = entry.getValue();
|
||||
try {
|
||||
JSONArray jsonArray = new JSONArray(item);
|
||||
List<Object> innerList = new ArrayList<>();
|
||||
for (int i = 0; i < jsonArray.length(); i++) {
|
||||
Object innerItem = jsonArray.get(i);
|
||||
if (innerItem instanceof JSONArray) {
|
||||
JSONArray innerArray = (JSONArray) innerItem;
|
||||
List<Object> innerArrayList = new ArrayList<>();
|
||||
for (int j = 0; j < innerArray.length(); j++) {
|
||||
innerArrayList.add(innerArray.get(j));
|
||||
}
|
||||
innerList.add(innerArrayList);
|
||||
} else {
|
||||
innerList.add(innerItem);
|
||||
}
|
||||
}
|
||||
write.put(key, innerList);
|
||||
} catch (Exception e) {
|
||||
// scalar cases
|
||||
write.put(key, item);
|
||||
}
|
||||
}
|
||||
gen.writeStartObject();
|
||||
for (Map.Entry<Integer, Object> entry : write.entrySet()) {
|
||||
gen.writeFieldName(entry.getKey().toString());
|
||||
Object item = entry.getValue();
|
||||
if (item instanceof List) {
|
||||
gen.writeStartArray();
|
||||
for (Object innerItem : (List<?>) item) {
|
||||
if (innerItem instanceof List) {
|
||||
gen.writeStartArray();
|
||||
for (Object innerArrayItem : (List<?>) innerItem) {
|
||||
gen.writeObject(innerArrayItem);
|
||||
}
|
||||
gen.writeEndArray();
|
||||
} else {
|
||||
gen.writeObject(innerItem);
|
||||
}
|
||||
}
|
||||
gen.writeEndArray();
|
||||
} else {
|
||||
gen.writeString((String) item);
|
||||
}
|
||||
}
|
||||
gen.writeEndObject();
|
||||
}
|
||||
}
|
||||
public static class StackTraceSerializer extends JsonSerializer<String> {
|
||||
@Override
|
||||
public void serialize(String value, JsonGenerator gen, SerializerProvider serializers) throws IOException {
|
||||
JSONArray jsonArray = new JSONArray();
|
||||
for(String item : value.split("\n")) {
|
||||
jsonArray.put(item.replace("\"\"",""));
|
||||
}
|
||||
gen.writeRawValue(jsonArray.toString(2));
|
||||
}
|
||||
}
|
||||
}
|
||||
+570
@@ -0,0 +1,570 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.interceptor.data;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.nd4j.common.primitives.Pair;
|
||||
import org.nd4j.interceptor.InterceptorEnvironment;
|
||||
|
||||
import java.sql.*;
|
||||
import java.util.*;
|
||||
|
||||
public class OpLogEventComparator {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
if (args.length != 3) {
|
||||
System.out.println("Please provide two database file paths and an epsilon value as arguments.");
|
||||
return;
|
||||
}
|
||||
|
||||
String jdbcUrl1 = "jdbc:h2:file:" + args[0];
|
||||
String jdbcUrl2 = "jdbc:h2:file:" + args[1];
|
||||
compareLinesBySide(jdbcUrl1, jdbcUrl2,1e-12);
|
||||
double epsilon = Double.parseDouble(args[2]);
|
||||
|
||||
try {
|
||||
Map<String, List<OpDifference>> differences = findDifferences(jdbcUrl1, jdbcUrl2, epsilon);
|
||||
|
||||
if (!differences.isEmpty()) {
|
||||
System.out.println("Found differences:");
|
||||
for (Map.Entry<String, List<OpDifference>> entry : differences.entrySet()) {
|
||||
System.out.println("Line of code: " + entry.getKey());
|
||||
for (OpDifference diff : entry.getValue()) {
|
||||
System.out.println(" Difference Type: " + diff.getDifferenceType());
|
||||
System.out.println(" Op Name: " + (diff.getOpLog1() != null ? diff.getOpLog1().getOpName() : diff.getOpLog2().getOpName()));
|
||||
System.out.println(" Difference Value 1: " + diff.getDifferenceValue1());
|
||||
System.out.println(" Difference Value 2: " + diff.getDifferenceValue2());
|
||||
System.out.println(" Op Difference: " + diff.getOpDifference());
|
||||
System.out.println();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
System.out.println("No differences found for the same inputs within the specified epsilon.");
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
public static Map<String, List<OpDifference>> findDifferences(String jdbcUrl1, String jdbcUrl2, double epsilon) throws SQLException {
|
||||
String query = "SELECT id, sourceCodeLine, opName, inputs, outputs, stackTrace FROM OpLogEvent ORDER BY id";
|
||||
Map<String, List<OpLogEvent>> events1 = new LinkedHashMap<>();
|
||||
Map<String, List<OpLogEvent>> events2 = new LinkedHashMap<>();
|
||||
|
||||
try (Connection conn1 = DriverManager.getConnection(jdbcUrl1, InterceptorEnvironment.USER, InterceptorEnvironment.PASSWORD);
|
||||
Connection conn2 = DriverManager.getConnection(jdbcUrl2, InterceptorEnvironment.USER, InterceptorEnvironment.PASSWORD);
|
||||
Statement stmt1 = conn1.createStatement();
|
||||
Statement stmt2 = conn2.createStatement();
|
||||
ResultSet rs1 = stmt1.executeQuery(query);
|
||||
ResultSet rs2 = stmt2.executeQuery(query)) {
|
||||
|
||||
processResultSet(rs1, events1);
|
||||
processResultSet(rs2, events2);
|
||||
}
|
||||
|
||||
return compareOpLogArrays(events1, events2, epsilon);
|
||||
}
|
||||
|
||||
private static void processResultSet(ResultSet rs, Map<String, List<OpLogEvent>> events) throws SQLException {
|
||||
while (rs.next()) {
|
||||
OpLogEvent event = createOpLogEvent(rs);
|
||||
String sourceLine = event.getFirstNonExecutionCodeLine();
|
||||
events.computeIfAbsent(sourceLine, k -> new ArrayList<>()).add(event);
|
||||
}
|
||||
}
|
||||
|
||||
private static OpLogEvent createOpLogEvent(ResultSet rs) throws SQLException {
|
||||
return OpLogEvent.builder()
|
||||
.eventId(rs.getLong("id"))
|
||||
.firstNonExecutionCodeLine(rs.getString("sourceCodeLine"))
|
||||
.opName(rs.getString("opName"))
|
||||
.inputs(convertResult(rs.getArray("inputs").getArray()))
|
||||
.outputs(convertResult(rs.getArray("outputs").getArray()))
|
||||
.stackTrace(rs.getString("stackTrace"))
|
||||
.build();
|
||||
}
|
||||
|
||||
private static Map<Integer, String> convertResult(Object input) {
|
||||
Object[] inputArr = (Object[]) input;
|
||||
Map<Integer, String> ret = new LinkedHashMap<>();
|
||||
for (int i = 0; i < inputArr.length; i++) {
|
||||
ret.put(i, inputArr[i].toString());
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
private static Map<String, List<OpDifference>> compareOpLogArrays(Map<String, List<OpLogEvent>> events1, Map<String, List<OpLogEvent>> events2, double epsilon) {
|
||||
Map<String, List<OpDifference>> differences = new LinkedHashMap<>();
|
||||
Map<String, OpDifference> earliestDifferences = new LinkedHashMap<>();
|
||||
Map<String, OpDifference> earliestSignificantDifferences = new LinkedHashMap<>();
|
||||
|
||||
for (String line : events1.keySet()) {
|
||||
List<OpLogEvent> opLogEvents1 = events1.get(line);
|
||||
List<OpLogEvent> opLogEvents2 = events2.getOrDefault(line, new ArrayList<>());
|
||||
|
||||
List<OpDifference> lineDifferences = new ArrayList<>();
|
||||
OpDifference earliestDifference = null;
|
||||
OpDifference earliestSignificantDifference = null;
|
||||
|
||||
int minSize = Math.min(opLogEvents1.size(), opLogEvents2.size());
|
||||
for (int i = 0; i < minSize; i++) {
|
||||
OpLogEvent opLogEvent1 = opLogEvents1.get(i);
|
||||
OpLogEvent opLogEvent2 = opLogEvents2.get(i);
|
||||
|
||||
// Compare inputs
|
||||
OpDifference inputDifference = compareInputs(opLogEvent1.getInputs(), opLogEvent2.getInputs(), epsilon, opLogEvent1, opLogEvent2);
|
||||
if (isValidDifference(inputDifference) && isSignificantDifference(inputDifference, epsilon)) {
|
||||
lineDifferences.add(inputDifference);
|
||||
earliestDifference = updateEarliestDifference(earliestDifference, inputDifference);
|
||||
earliestSignificantDifference = updateEarliestDifference(earliestSignificantDifference, inputDifference);
|
||||
}
|
||||
|
||||
// Compare outputs
|
||||
OpDifference outputDifference = compareOutputs(opLogEvent1.getOutputs(), opLogEvent2.getOutputs(), epsilon, opLogEvent1, opLogEvent2);
|
||||
if (isValidDifference(outputDifference) && isSignificantDifference(outputDifference, epsilon)) {
|
||||
lineDifferences.add(outputDifference);
|
||||
earliestDifference = updateEarliestDifference(earliestDifference, outputDifference);
|
||||
earliestSignificantDifference = updateEarliestDifference(earliestSignificantDifference, outputDifference);
|
||||
}
|
||||
}
|
||||
|
||||
if (!lineDifferences.isEmpty()) {
|
||||
differences.put(line, lineDifferences);
|
||||
}
|
||||
if (earliestDifference != null) {
|
||||
earliestDifferences.put(line, earliestDifference);
|
||||
}
|
||||
if (earliestSignificantDifference != null) {
|
||||
earliestSignificantDifferences.put(line, earliestSignificantDifference);
|
||||
}
|
||||
}
|
||||
|
||||
// Check for lines in events2 that are not in events1
|
||||
for (String line : events2.keySet()) {
|
||||
if (!events1.containsKey(line)) {
|
||||
List<OpLogEvent> opLogEvents2 = events2.get(line);
|
||||
OpDifference missingLineDifference = OpDifference.builder()
|
||||
.opLog1(null)
|
||||
.opLog2(opLogEvents2.get(0))
|
||||
.differenceType("missing_line")
|
||||
.differenceValue1("null")
|
||||
.differenceValue2(line)
|
||||
.opDifference(opLogEvents2.size())
|
||||
.build();
|
||||
if (isValidDifference(missingLineDifference) && isSignificantDifference(missingLineDifference, epsilon)) {
|
||||
differences.put(line, Collections.singletonList(missingLineDifference));
|
||||
earliestDifferences.put(line, missingLineDifference);
|
||||
earliestSignificantDifferences.put(line, missingLineDifference);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove any invalid or insignificant elements from the final differences result
|
||||
differences.entrySet().removeIf(entry -> entry.getValue().isEmpty());
|
||||
|
||||
// Print out the earliest difference for each line
|
||||
System.out.println("Earliest differences per line of code:");
|
||||
printDifferences(earliestDifferences);
|
||||
|
||||
// Create a final sorted list of lines of code with the earliest difference
|
||||
List<Map.Entry<String, OpDifference>> sortedEarliestDifferences = sortDifferences(earliestDifferences);
|
||||
|
||||
// Print the sorted list of earliest differences
|
||||
System.out.println("\nSorted list of lines of code with the earliest difference:");
|
||||
printSortedDifferences(sortedEarliestDifferences);
|
||||
|
||||
// Create and print a sorted list of significant differences
|
||||
List<Map.Entry<String, OpDifference>> sortedSignificantDifferences = sortDifferences(earliestSignificantDifferences);
|
||||
System.out.println("\nSorted list of lines of code with significant differences:");
|
||||
printSortedDifferences(sortedSignificantDifferences);
|
||||
|
||||
return differences;
|
||||
}
|
||||
|
||||
|
||||
private static Map<String, List<OpLogEvent>> loadEvents(String jdbcUrl) throws SQLException {
|
||||
Map<String, List<OpLogEvent>> events = new HashMap<>();
|
||||
String query = "SELECT id, sourceCodeLine, opName, inputs, outputs, stackTrace FROM OpLogEvent ORDER BY id";
|
||||
|
||||
try (Connection conn = DriverManager.getConnection(jdbcUrl, InterceptorEnvironment.USER, InterceptorEnvironment.PASSWORD);
|
||||
Statement stmt = conn.createStatement();
|
||||
ResultSet rs = stmt.executeQuery(query)) {
|
||||
processResultSet(rs, events);
|
||||
}
|
||||
|
||||
return events;
|
||||
}
|
||||
|
||||
private static class LineComparison {
|
||||
String line;
|
||||
List<OpLogEvent> events1;
|
||||
List<OpLogEvent> events2;
|
||||
String difference;
|
||||
|
||||
LineComparison(String line, List<OpLogEvent> events1, List<OpLogEvent> events2, String difference) {
|
||||
this.line = line;
|
||||
this.events1 = events1;
|
||||
this.events2 = events2;
|
||||
this.difference = difference;
|
||||
}
|
||||
|
||||
long getEarliestEventTime() {
|
||||
long time1 = events1.isEmpty() ? Long.MAX_VALUE : events1.get(0).getEventId();
|
||||
long time2 = events2.isEmpty() ? Long.MAX_VALUE : events2.get(0).getEventId();
|
||||
return Math.min(time1, time2);
|
||||
}
|
||||
}
|
||||
|
||||
public static void compareLinesBySide(String jdbcUrl1, String jdbcUrl2, double epsilon) throws SQLException {
|
||||
Map<String, List<OpLogEvent>> events1 = loadAndNormalizeEvents(jdbcUrl1);
|
||||
Map<String, List<OpLogEvent>> events2 = loadAndNormalizeEvents(jdbcUrl2);
|
||||
|
||||
Set<String> allLines = new LinkedHashSet<>(events1.keySet());
|
||||
allLines.addAll(events2.keySet());
|
||||
|
||||
List<LineComparison> comparisons = new ArrayList<>();
|
||||
for (String line : allLines) {
|
||||
List<OpLogEvent> lineEvents1 = events1.getOrDefault(line, Collections.emptyList());
|
||||
List<OpLogEvent> lineEvents2 = events2.getOrDefault(line, Collections.emptyList());
|
||||
Pair<String,Integer> difference = findSignificantDifference(lineEvents1, lineEvents2, epsilon);
|
||||
if (difference != null) {
|
||||
comparisons.add(new LineComparison(line, lineEvents1, lineEvents2, difference.getKey()));
|
||||
}
|
||||
}
|
||||
|
||||
if (comparisons.isEmpty()) {
|
||||
System.out.println("No significant differences found.");
|
||||
return;
|
||||
}
|
||||
|
||||
comparisons.sort(Comparator.comparing(LineComparison::getEarliestEventTime));
|
||||
|
||||
System.out.println("Side-by-side comparison of lines with significant differences (sorted by earliest event time):");
|
||||
System.out.println("----------------------------------------------------");
|
||||
System.out.printf("%-50s | %-50s | %-30s%n", "Database 1", "Database 2", "Difference");
|
||||
System.out.println("----------------------------------------------------");
|
||||
|
||||
for (LineComparison comparison : comparisons) {
|
||||
String summary1 = summarizeEvents(comparison.events1);
|
||||
String summary2 = summarizeEvents(comparison.events2);
|
||||
|
||||
System.out.printf("%-50s | %-50s | %-30s%n", summary1, summary2, comparison.difference);
|
||||
System.out.println("Line: " + comparison.line);
|
||||
System.out.println("----------------------------------------------------");
|
||||
}
|
||||
}
|
||||
|
||||
private static String summarizeEvents(List<OpLogEvent> events) {
|
||||
if (events.isEmpty()) {
|
||||
return "<No events>";
|
||||
}
|
||||
|
||||
int count = events.size();
|
||||
String firstOpName = events.get(0).getOpName();
|
||||
long earliestEventId = events.get(0).getEventId();
|
||||
return String.format("%d events, first: %s (ID: %d)", count, firstOpName, earliestEventId);
|
||||
}
|
||||
|
||||
private static Pair<String,Integer> findSignificantDifference(List<OpLogEvent> events1, List<OpLogEvent> events2, double epsilon) {
|
||||
int minSize = Math.min(events1.size(), events2.size());
|
||||
|
||||
for (int i = 0; i < minSize; i++) {
|
||||
OpLogEvent e1 = events1.get(i);
|
||||
OpLogEvent e2 = events2.get(i);
|
||||
|
||||
|
||||
String inputDiff = compareWithEpsilon(e1.getInputs(), e2.getInputs(), epsilon);
|
||||
if (inputDiff != null) {
|
||||
return Pair.of("Inputs differ: " + inputDiff,i);
|
||||
}
|
||||
|
||||
String outputDiff = compareWithEpsilon(e1.getOutputs(), e2.getOutputs(), epsilon);
|
||||
if (outputDiff != null) {
|
||||
return Pair.of("Outputs differ: " + outputDiff,i);
|
||||
}
|
||||
}
|
||||
|
||||
return null; // No significant difference found
|
||||
}
|
||||
|
||||
private static String compareWithEpsilon(Map<Integer, String> map1, Map<Integer, String> map2, double epsilon) {
|
||||
for (Integer key : map1.keySet()) {
|
||||
if (!map2.containsKey(key)) continue; // Ignore keys not present in both maps
|
||||
|
||||
String value1 = map1.get(key);
|
||||
String value2 = map2.get(key);
|
||||
|
||||
//dup bug, ignore
|
||||
if(value1.contains("[]") || value2.contains("[]")) continue;
|
||||
try {
|
||||
JSONArray arr1 = new JSONArray(value1);
|
||||
JSONArray arr2 = new JSONArray(value2);
|
||||
|
||||
String arrayDiff = compareArraysWithEpsilon(arr1, arr2, epsilon);
|
||||
if (arrayDiff != null) {
|
||||
return "Key " + key + ": " + arrayDiff;
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
// If not a JSON array, compare as individual values
|
||||
try {
|
||||
double d1 = Double.parseDouble(value1);
|
||||
double d2 = Double.parseDouble(value2);
|
||||
|
||||
if (Math.abs(d1 - d2) > epsilon) {
|
||||
return String.format("Key %d: %f vs %f", key, d1, d2);
|
||||
}
|
||||
} catch (NumberFormatException nfe) {
|
||||
// If values are not numbers, compare them as strings
|
||||
if (!value1.equals(value2)) {
|
||||
return String.format("Key %d: %s vs %s", key, value1, value2);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null; // No significant difference found
|
||||
}
|
||||
|
||||
private static String compareArraysWithEpsilon(JSONArray arr1, JSONArray arr2, double epsilon) throws JSONException {
|
||||
if (arr1.length() != arr2.length()) {
|
||||
return "Array lengths differ: " + arr1.length() + " vs " + arr2.length();
|
||||
}
|
||||
|
||||
for (int i = 0; i < arr1.length(); i++) {
|
||||
Object val1 = arr1.get(i);
|
||||
Object val2 = arr2.get(i);
|
||||
|
||||
if (val1 instanceof JSONArray && val2 instanceof JSONArray) {
|
||||
String nestedDiff = compareArraysWithEpsilon((JSONArray) val1, (JSONArray) val2, epsilon);
|
||||
if (nestedDiff != null) {
|
||||
return "Nested array at index " + i + ": " + nestedDiff;
|
||||
}
|
||||
} else if (val1 instanceof Number && val2 instanceof Number) {
|
||||
double d1 = ((Number) val1).doubleValue();
|
||||
double d2 = ((Number) val2).doubleValue();
|
||||
if (Math.abs(d1 - d2) > epsilon) {
|
||||
return String.format("Index %d: %f vs %f", i, d1, d2);
|
||||
}
|
||||
} else if (!val1.equals(val2)) {
|
||||
return String.format("Index %d: %s vs %s", i, val1, val2);
|
||||
}
|
||||
}
|
||||
|
||||
return null; // No significant difference found
|
||||
}
|
||||
private static Map<String, List<OpLogEvent>> loadAndNormalizeEvents(String jdbcUrl) throws SQLException {
|
||||
Map<String, List<OpLogEvent>> events = new LinkedHashMap<>();
|
||||
String query = "SELECT id, sourceCodeLine, opName, inputs, outputs, stackTrace FROM OpLogEvent ORDER BY id";
|
||||
|
||||
try (Connection conn = DriverManager.getConnection(jdbcUrl, InterceptorEnvironment.USER, InterceptorEnvironment.PASSWORD);
|
||||
Statement stmt = conn.createStatement();
|
||||
ResultSet rs = stmt.executeQuery(query)) {
|
||||
processResultSet(rs, events);
|
||||
}
|
||||
|
||||
// Normalize and deduplicate events
|
||||
for (Map.Entry<String, List<OpLogEvent>> entry : events.entrySet()) {
|
||||
List<OpLogEvent> normalizedEvents = normalizeAndDeduplicateEvents(entry.getValue());
|
||||
entry.setValue(normalizedEvents);
|
||||
}
|
||||
|
||||
return events;
|
||||
}
|
||||
|
||||
private static List<OpLogEvent> normalizeAndDeduplicateEvents(List<OpLogEvent> events) {
|
||||
Set<OpLogEvent> normalizedSet = new LinkedHashSet<>();
|
||||
for (OpLogEvent event : events) {
|
||||
OpLogEvent normalizedEvent = OpLogEvent.builder()
|
||||
.eventId(0L) // Set eventId to 0
|
||||
.firstNonExecutionCodeLine(event.getFirstNonExecutionCodeLine())
|
||||
.opName(event.getOpName())
|
||||
.inputs(event.getInputs())
|
||||
.outputs(event.getOutputs())
|
||||
.stackTrace(event.getStackTrace())
|
||||
.build();
|
||||
normalizedSet.add(normalizedEvent);
|
||||
}
|
||||
return new ArrayList<>(normalizedSet);
|
||||
}
|
||||
|
||||
|
||||
|
||||
private static OpDifference compareInputs(Map<Integer, String> inputs1, Map<Integer, String> inputs2, double epsilon, OpLogEvent opLogEvent1, OpLogEvent opLogEvent2) {
|
||||
for (int j = 0; j < Math.min(inputs1.size(), inputs2.size()); j++) {
|
||||
JSONArray jsonArray1 = new JSONArray(inputs1.get(j));
|
||||
JSONArray jsonArray2 = new JSONArray(inputs2.get(j));
|
||||
JSONComparisonResult result = compareJSONArraysWithEpsilon(jsonArray1, jsonArray2, epsilon);
|
||||
if (!result.isSame()) {
|
||||
return OpDifference.builder()
|
||||
.opLog1(opLogEvent1)
|
||||
.opLog2(opLogEvent2)
|
||||
.differenceType("inputs")
|
||||
.differenceValue1(String.valueOf(result.getFirstValue()))
|
||||
.differenceValue2(String.valueOf(result.getSecondValue()))
|
||||
.opDifference(j)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static OpDifference compareOutputs(Map<Integer, String> outputs1, Map<Integer, String> outputs2, double epsilon, OpLogEvent opLogEvent1, OpLogEvent opLogEvent2) {
|
||||
for (int j = 0; j < Math.min(outputs1.size(), outputs2.size()); j++) {
|
||||
Object cast1 = parseOutput(outputs1.get(j));
|
||||
Object cast2 = parseOutput(outputs2.get(j));
|
||||
|
||||
JSONArray casted1 = (JSONArray) cast1;
|
||||
JSONArray casted2 = (JSONArray) cast2;
|
||||
|
||||
JSONComparisonResult result = compareJSONArraysWithEpsilon(casted1, casted2, epsilon);
|
||||
if (!result.isSame()) {
|
||||
return OpDifference.builder()
|
||||
.opLog1(opLogEvent1)
|
||||
.opLog2(opLogEvent2)
|
||||
.differenceType("outputs")
|
||||
.differenceValue1(String.valueOf(result.getFirstValue()))
|
||||
.differenceValue2(String.valueOf(result.getSecondValue()))
|
||||
.opDifference(result.getIndex())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static Object parseOutput(String output) {
|
||||
if (output.matches("-*\\d+\\.\\d+")) {
|
||||
return new JSONArray(new double[]{Double.parseDouble(output)});
|
||||
} else {
|
||||
return new JSONArray(output);
|
||||
}
|
||||
}
|
||||
|
||||
private static JSONComparisonResult compareJSONArraysWithEpsilon(JSONArray arr1, JSONArray arr2, double epsilon) {
|
||||
if (arr1.length() != arr2.length()) {
|
||||
return JSONComparisonResult.builder().same(false).index(-1).build();
|
||||
}
|
||||
|
||||
for (int i = 0; i < arr1.length(); i++) {
|
||||
if (arr1.get(i) instanceof JSONArray && arr2.get(i) instanceof JSONArray) {
|
||||
JSONComparisonResult result = compareJSONArraysWithEpsilon(arr1.getJSONArray(i), arr2.getJSONArray(i), epsilon);
|
||||
if (!result.isSame()) {
|
||||
return result;
|
||||
}
|
||||
} else {
|
||||
double val1 = arr1.getDouble(i);
|
||||
double val2 = arr2.getDouble(i);
|
||||
if (Math.abs(val1 - val2) > epsilon) {
|
||||
return JSONComparisonResult.builder()
|
||||
.same(false)
|
||||
.index(i)
|
||||
.firstValue(val1)
|
||||
.secondValue(val2)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return JSONComparisonResult.builder().same(true).build();
|
||||
}
|
||||
|
||||
private static boolean isValidDifference(OpDifference diff) {
|
||||
if (diff == null) {
|
||||
return false;
|
||||
}
|
||||
if (diff.getOpLog1() == null || diff.getOpLog2() == null) {
|
||||
return false;
|
||||
}
|
||||
if (diff.getOpDifference() == -1) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static boolean isSignificantDifference(OpDifference diff, double epsilon) {
|
||||
if (!isValidDifference(diff)) {
|
||||
return false;
|
||||
}
|
||||
if (diff.getDifferenceType().equals("size") || diff.getDifferenceType().equals("missing_line")) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
double value1 = Double.parseDouble(diff.getDifferenceValue1());
|
||||
double value2 = Double.parseDouble(diff.getDifferenceValue2());
|
||||
return Math.abs(value1 - value2) > epsilon;
|
||||
} catch (NumberFormatException e) {
|
||||
// If we can't parse the values as doubles, consider it significant
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static OpDifference updateEarliestDifference(OpDifference currentEarliest, OpDifference newDifference) {
|
||||
if (currentEarliest == null) {
|
||||
return newDifference;
|
||||
}
|
||||
|
||||
long currentEarliestTime = getEarliestTime(currentEarliest);
|
||||
long newDifferenceTime = getEarliestTime(newDifference);
|
||||
|
||||
return newDifferenceTime < currentEarliestTime ? newDifference : currentEarliest;
|
||||
}
|
||||
|
||||
private static long getEarliestTime(OpDifference diff) {
|
||||
return Math.min(
|
||||
diff.getOpLog1() != null ? diff.getOpLog1().getEventId() : Long.MAX_VALUE,
|
||||
diff.getOpLog2() != null ? diff.getOpLog2().getEventId() : Long.MAX_VALUE
|
||||
);
|
||||
}
|
||||
|
||||
private static List<Map.Entry<String, OpDifference>> sortDifferences(Map<String, OpDifference> differences) {
|
||||
List<Map.Entry<String, OpDifference>> sortedDifferences = new ArrayList<>(differences.entrySet());
|
||||
sortedDifferences.sort((e1, e2) -> {
|
||||
long time1 = getEarliestTime(e1.getValue());
|
||||
long time2 = getEarliestTime(e2.getValue());
|
||||
return Long.compare(time1, time2);
|
||||
});
|
||||
return sortedDifferences;
|
||||
}
|
||||
|
||||
private static void printDifferences(Map<String, OpDifference> differences) {
|
||||
for (Map.Entry<String, OpDifference> entry : differences.entrySet()) {
|
||||
System.out.println("Line: " + entry.getKey());
|
||||
OpDifference diff = entry.getValue();
|
||||
System.out.println(" Earliest Difference Type: " + diff.getDifferenceType());
|
||||
System.out.println(" Earliest Event ID: " + getEarliestTime(diff));
|
||||
System.out.println();
|
||||
}
|
||||
}
|
||||
|
||||
private static void printSortedDifferences(List<Map.Entry<String, OpDifference>> sortedDifferences) {
|
||||
for (Map.Entry<String, OpDifference> entry : sortedDifferences) {
|
||||
String line = entry.getKey();
|
||||
OpDifference diff = entry.getValue();
|
||||
long earliestTime = getEarliestTime(diff);
|
||||
System.out.println("Line: " + line);
|
||||
System.out.println(" Earliest Difference Type: " + diff.getDifferenceType());
|
||||
System.out.println(" Earliest Event ID: " + earliestTime);
|
||||
System.out.println(" Op Name: " + (diff.getOpLog1() != null ? diff.getOpLog1().getOpName() : diff.getOpLog2().getOpName()));
|
||||
System.out.println(" Difference Value 1: " + diff.getDifferenceValue1());
|
||||
System.out.println(" Difference Value 2: " + diff.getDifferenceValue2());
|
||||
System.out.println();
|
||||
}
|
||||
}
|
||||
}
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.interceptor.data;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
|
||||
import org.nd4j.shade.jackson.databind.annotation.JsonSerialize;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@JsonSerialize(using = OpLogEventWriteSerializer.class)
|
||||
public class OpLogEventWrite {
|
||||
|
||||
public String opName;
|
||||
|
||||
@Builder.Default
|
||||
@JsonSerialize(using = OpLogEvent.InputOutputSerializer.class)
|
||||
public Map<Integer,String> inputs = new LinkedHashMap<>();
|
||||
|
||||
@Builder.Default
|
||||
@JsonSerialize(using = OpLogEvent.InputOutputSerializer.class)
|
||||
public Map<Integer,String> outputs = new LinkedHashMap<>();
|
||||
|
||||
@Builder.Default
|
||||
@JsonSerialize(using = OpLogEvent.StackTraceSerializer.class)
|
||||
public String[] stackTrace = new String[0];
|
||||
|
||||
public String firstNonExecutionCodeLine;
|
||||
|
||||
|
||||
public long eventId;
|
||||
|
||||
public OpLogEventWrite(OpLogEvent opLogEvent) {
|
||||
this.opName = opLogEvent.getOpName();
|
||||
this.inputs = opLogEvent.getInputs();
|
||||
this.outputs = opLogEvent.getOutputs();
|
||||
this.stackTrace = opLogEvent.getStackTrace().split("\n");
|
||||
this.eventId = opLogEvent.getEventId();
|
||||
this.firstNonExecutionCodeLine = opLogEvent.getFirstNonExecutionCodeLine();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
package org.nd4j.interceptor.data;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.nd4j.shade.jackson.core.JsonGenerator;
|
||||
import org.nd4j.shade.jackson.databind.JsonSerializer;
|
||||
import org.nd4j.shade.jackson.databind.ObjectMapper;
|
||||
import org.nd4j.shade.jackson.databind.SerializationFeature;
|
||||
import org.nd4j.shade.jackson.databind.SerializerProvider;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class OpLogEventWriteSerializer extends JsonSerializer<OpLogEvent> {
|
||||
@Override
|
||||
public void serialize(OpLogEvent value, JsonGenerator gen, SerializerProvider serializers) throws IOException {
|
||||
gen.useDefaultPrettyPrinter();
|
||||
|
||||
gen.writeStartObject();
|
||||
|
||||
gen.writeFieldName("opName");
|
||||
gen.writeString(value.getOpName());
|
||||
|
||||
gen.writeFieldName("inputs");
|
||||
serializeInputOutput(value.getInputs(), gen);
|
||||
|
||||
gen.writeFieldName("outputs");
|
||||
serializeInputOutput(value.getOutputs(), gen);
|
||||
|
||||
gen.writeFieldName("stackTrace");
|
||||
serializeStackTrace(value.getStackTrace().split("\n"), gen);
|
||||
|
||||
gen.writeFieldName("eventId");
|
||||
gen.writeNumber(value.getEventId());
|
||||
gen.writeEndObject();
|
||||
}
|
||||
|
||||
private void serializeInputOutput(Map<Integer, String> valuesMap, JsonGenerator gen) throws IOException {
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
|
||||
Map<String, Object> write = new LinkedHashMap<>();
|
||||
for (Map.Entry<Integer, String> entry : valuesMap.entrySet()) {
|
||||
String item = entry.getValue();
|
||||
try {
|
||||
JSONArray jsonArray = new JSONArray(item);
|
||||
write.put(String.valueOf(entry.getKey()), jsonArray.toString(2));
|
||||
} catch (Exception e) {
|
||||
// scalar cases
|
||||
write.put(String.valueOf(entry.getKey()), item);
|
||||
}
|
||||
}
|
||||
gen.writeStartObject();
|
||||
for (Map.Entry<String, Object> entry : write.entrySet()) {
|
||||
gen.writeFieldName(entry.getKey());
|
||||
if (entry.getValue() instanceof Map) {
|
||||
gen.writeStartObject();
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> map = (Map<String, Object>) entry.getValue();
|
||||
for (Map.Entry<String, Object> mapEntry : map.entrySet()) {
|
||||
gen.writeFieldName(mapEntry.getKey());
|
||||
if (mapEntry.getValue() instanceof Map) {
|
||||
gen.writeStartObject();
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> innerMap = (Map<String, Object>) mapEntry.getValue();
|
||||
for (Map.Entry<String, Object> innerEntry : innerMap.entrySet()) {
|
||||
gen.writeFieldName(innerEntry.getKey());
|
||||
gen.writeNumber(((Double) innerEntry.getValue()).doubleValue());
|
||||
}
|
||||
gen.writeEndObject();
|
||||
} else {
|
||||
gen.writeString((String) mapEntry.getValue());
|
||||
}
|
||||
}
|
||||
gen.writeEndObject();
|
||||
} else {
|
||||
gen.writeString((String) entry.getValue());
|
||||
}
|
||||
}
|
||||
gen.writeEndObject();
|
||||
}
|
||||
|
||||
private void serializeStackTrace(String[] stackTrace, JsonGenerator gen) throws IOException {
|
||||
if(stackTrace.length == 1) {
|
||||
JSONArray jsonArray = new JSONArray(stackTrace[0]);
|
||||
gen.writeRawValue(jsonArray.toString(2));
|
||||
} else {
|
||||
JSONArray jsonArray = new JSONArray();
|
||||
for (String item : stackTrace) {
|
||||
jsonArray.put(item);
|
||||
}
|
||||
gen.writeRawValue(jsonArray.toString(2));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.interceptor.data;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@AllArgsConstructor
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@Builder
|
||||
public class SourceCodeOpEvent {
|
||||
private Map<String,List<OpLogEvent>> opLogEvents;
|
||||
}
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
package org.nd4j.interceptor.parser;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.AllArgsConstructor;
|
||||
import org.nd4j.shade.jackson.databind.ObjectMapper;
|
||||
import org.nd4j.shade.jackson.databind.SerializationFeature;
|
||||
import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize;
|
||||
import org.nd4j.shade.jackson.databind.annotation.JsonSerialize;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@JsonSerialize(using = SourceCodeIndexComparatorSerializer.class)
|
||||
@JsonDeserialize(using = SourceCodeIndexComparatorDeserializer.class)
|
||||
|
||||
public class SourceCodeIndexComparator {
|
||||
private SourceCodeIndexer index1;
|
||||
private SourceCodeIndexer index2;
|
||||
private ObjectMapper objectMapper;
|
||||
private Map<SourceCodeLine, SourceCodeLine> comparisonResult;
|
||||
private Map<SourceCodeLine, SourceCodeLine> reverseComparisonResult;
|
||||
|
||||
public SourceCodeIndexComparator(File indexFile1, File indexFile2) throws IOException {
|
||||
objectMapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
|
||||
this.index1 = objectMapper.readValue(indexFile1, SourceCodeIndexer.class);
|
||||
this.index2 = objectMapper.readValue(indexFile2, SourceCodeIndexer.class);
|
||||
}
|
||||
|
||||
public void compareIndexes() {
|
||||
comparisonResult = new HashMap<>();
|
||||
reverseComparisonResult = new HashMap<>();
|
||||
|
||||
for (String className : index1.getIndex().rowKeySet()) {
|
||||
for (Integer lineNumber : index1.getIndex().columnKeySet()) {
|
||||
SourceCodeLine line1 = index1.getSourceCodeLine(className, lineNumber);
|
||||
SourceCodeLine line2 = index2.getSourceCodeLine(className,lineNumber);
|
||||
|
||||
if (line2 != null && line1.getLine().equals(line2.getLine())) {
|
||||
comparisonResult.put(line1, line2);
|
||||
reverseComparisonResult.put(line2, line1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void saveComparisonResult(String fileName) {
|
||||
try {
|
||||
objectMapper.writeValue(new File(fileName), comparisonResult);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package org.nd4j.interceptor.parser;
|
||||
|
||||
import org.nd4j.shade.jackson.core.JsonParser;
|
||||
import org.nd4j.shade.jackson.core.type.TypeReference;
|
||||
import org.nd4j.shade.jackson.databind.DeserializationContext;
|
||||
import org.nd4j.shade.jackson.databind.JsonDeserializer;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
|
||||
public class SourceCodeIndexComparatorDeserializer extends JsonDeserializer<SourceCodeIndexComparator> {
|
||||
@Override
|
||||
public SourceCodeIndexComparator deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
|
||||
SourceCodeIndexer index1 = p.readValueAs(SourceCodeIndexer.class);
|
||||
SourceCodeIndexer index2 = p.readValueAs(SourceCodeIndexer.class);
|
||||
Map<SourceCodeLine, SourceCodeLine> comparisonResult = p.readValueAs(new TypeReference<Map<SourceCodeLine, SourceCodeLine>>() {});
|
||||
Map<SourceCodeLine, SourceCodeLine> reverseComparisonResult = p.readValueAs(new TypeReference<Map<SourceCodeLine, SourceCodeLine>>() {});
|
||||
|
||||
return SourceCodeIndexComparator.builder()
|
||||
.index1(index1)
|
||||
.index2(index2)
|
||||
.comparisonResult(comparisonResult)
|
||||
.reverseComparisonResult(reverseComparisonResult)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package org.nd4j.interceptor.parser;
|
||||
|
||||
import org.nd4j.shade.jackson.core.JsonGenerator;
|
||||
import org.nd4j.shade.jackson.databind.JsonSerializer;
|
||||
import org.nd4j.shade.jackson.databind.SerializerProvider;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class SourceCodeIndexComparatorSerializer extends JsonSerializer<SourceCodeIndexComparator> {
|
||||
@Override
|
||||
public void serialize(SourceCodeIndexComparator value, JsonGenerator gen, SerializerProvider serializers) throws IOException {
|
||||
gen.writeStartObject();
|
||||
gen.writeObjectField("index1", value.getIndex1());
|
||||
gen.writeObjectField("index2", value.getIndex2());
|
||||
gen.writeObjectField("comparisonResult", value.getComparisonResult());
|
||||
gen.writeObjectField("reverseComparisonResult", value.getReverseComparisonResult());
|
||||
gen.writeEndObject();
|
||||
}
|
||||
}
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.interceptor.parser;
|
||||
|
||||
import com.github.javaparser.StaticJavaParser;
|
||||
import com.github.javaparser.ast.body.MethodDeclaration;
|
||||
import com.github.javaparser.ast.stmt.Statement;
|
||||
import com.github.javaparser.symbolsolver.JavaSymbolSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.JavaParserTypeSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.ReflectionTypeSolver;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import org.nd4j.shade.guava.collect.HashBasedTable;
|
||||
import org.nd4j.shade.guava.collect.Table;
|
||||
import org.nd4j.shade.jackson.databind.ObjectMapper;
|
||||
import org.nd4j.shade.jackson.databind.SerializationFeature;
|
||||
import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize;
|
||||
import org.nd4j.shade.jackson.databind.annotation.JsonSerialize;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.sql.*;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.nd4j.interceptor.InterceptorEnvironment.PASSWORD;
|
||||
import static org.nd4j.interceptor.InterceptorEnvironment.USER;
|
||||
|
||||
@NoArgsConstructor
|
||||
@Data
|
||||
@JsonSerialize(using = SourceCodeIndexerSerializer.class)
|
||||
@JsonDeserialize(using = SourceCodeIndexerDeserializer.class)
|
||||
public class SourceCodeIndexer {
|
||||
|
||||
private Table<String,Integer, SourceCodeLine> index = HashBasedTable.create();
|
||||
|
||||
public SourceCodeIndexer(File dl4jRoot,String dbPath) {
|
||||
initSourceRoot(dl4jRoot,dbPath);
|
||||
}
|
||||
|
||||
|
||||
public SourceCodeLine getSourceCodeLine(String fullClassName, int lineNumber) {
|
||||
return index.get(fullClassName, lineNumber);
|
||||
}
|
||||
|
||||
public Set<String> getClasses() {
|
||||
return index.rowKeySet();
|
||||
}
|
||||
|
||||
|
||||
public void persistToOpLog(String dbPath) {
|
||||
String jdbcUrl = "jdbc:h2:file:" + dbPath + ";";
|
||||
Set<SourceCodeLine> lines = index.values().stream().collect(Collectors.toSet());
|
||||
System.out.println("Finished indexing.");
|
||||
String insertQuery = "INSERT INTO SourceCodeLine(className, lineNumber, line, packageName, fileName, lastUpdated) VALUES (?, ?, ?, ?, ?, ?)";
|
||||
String updateQuery = "UPDATE SourceCodeLine SET line = ?, lastUpdated = ? WHERE className = ? AND lineNumber = ?";
|
||||
|
||||
try (Connection conn = DriverManager.getConnection(jdbcUrl, USER, PASSWORD)) {
|
||||
conn.setAutoCommit(false);
|
||||
|
||||
try (PreparedStatement insertStmt = conn.prepareStatement(insertQuery);
|
||||
PreparedStatement updateStmt = conn.prepareStatement(updateQuery)) {
|
||||
|
||||
for (SourceCodeLine line : lines) {
|
||||
// Check if the line already exists in the database
|
||||
String selectQuery = "SELECT * FROM SourceCodeLine WHERE className = ? AND lineNumber = ?";
|
||||
try (PreparedStatement selectStmt = conn.prepareStatement(selectQuery)) {
|
||||
selectStmt.setString(1, line.getClassName());
|
||||
selectStmt.setInt(2, line.getLineNumber());
|
||||
ResultSet resultSet = selectStmt.executeQuery();
|
||||
|
||||
if (resultSet.next()) {
|
||||
// Line already exists, check if it needs to be updated
|
||||
String existingLine = resultSet.getString("line");
|
||||
Timestamp existingTimestamp = resultSet.getTimestamp("lastUpdated");
|
||||
File file = new File(line.getFileName());
|
||||
long fileLastModified = file.lastModified();
|
||||
|
||||
if (!existingLine.equals(line.getLine()) || existingTimestamp.getTime() < fileLastModified) {
|
||||
// Line content has changed or the file has been updated, update the line
|
||||
updateStmt.setString(1, line.getLine());
|
||||
updateStmt.setTimestamp(2, new Timestamp(fileLastModified));
|
||||
updateStmt.setString(3, line.getClassName());
|
||||
updateStmt.setInt(4, line.getLineNumber());
|
||||
updateStmt.addBatch();
|
||||
}
|
||||
} else {
|
||||
// Line doesn't exist, insert a new line
|
||||
insertStmt.setString(1, line.getClassName());
|
||||
insertStmt.setInt(2, line.getLineNumber());
|
||||
insertStmt.setString(3, line.getLine());
|
||||
insertStmt.setString(4, line.getPackageName());
|
||||
insertStmt.setString(5, line.getFileName());
|
||||
insertStmt.setTimestamp(6, new Timestamp(new File(line.getFileName()).lastModified()));
|
||||
insertStmt.addBatch();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
insertStmt.executeBatch();
|
||||
updateStmt.executeBatch();
|
||||
conn.commit();
|
||||
} catch (SQLException e) {
|
||||
conn.rollback();
|
||||
throw new RuntimeException("Failed to persist source code index to OpLog", e);
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException("Failed to persist source code index to OpLog", e);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@SneakyThrows
|
||||
public void initSourceRoot(File nd4jApiRootDir,String dbPath) {
|
||||
CombinedTypeSolver typeSolver = new CombinedTypeSolver();
|
||||
typeSolver.add(new ReflectionTypeSolver(false));
|
||||
typeSolver.add(new JavaParserTypeSolver(nd4jApiRootDir));
|
||||
JavaSymbolSolver symbolSolver = new JavaSymbolSolver(typeSolver);
|
||||
StaticJavaParser.getConfiguration().setSymbolResolver(symbolSolver);
|
||||
|
||||
String jdbcUrl = "jdbc:h2:file:" + dbPath + ";";
|
||||
String query = "SELECT * FROM SourceCodeLine WHERE fileName = ?";
|
||||
|
||||
try (Connection conn = DriverManager.getConnection(jdbcUrl, USER, PASSWORD)) {
|
||||
Files.walk(nd4jApiRootDir.toPath()).parallel()
|
||||
.map(Path::toFile)
|
||||
.filter(file -> !file.isDirectory() && file.getName().endsWith(".java"))
|
||||
.forEach(file -> {
|
||||
try (PreparedStatement stmt = conn.prepareStatement(query)) {
|
||||
stmt.setString(1, file.getAbsolutePath());
|
||||
ResultSet resultSet = stmt.executeQuery();
|
||||
if (resultSet.next()) {
|
||||
Timestamp lastUpdatedTimestamp = resultSet.getTimestamp("lastUpdated");
|
||||
long lastUpdatedTime = lastUpdatedTimestamp != null ? lastUpdatedTimestamp.getTime() : 0;
|
||||
if (file.lastModified() <= lastUpdatedTime) {
|
||||
// Skip indexing this file if it hasn't been updated since the last indexing
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException("Failed to check file timestamp in the database", e);
|
||||
}
|
||||
indexFile(file);
|
||||
});
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException("Failed to establish database connection", e);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Failed to walk the directory", e);
|
||||
}
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
private void indexFile(File javaSourceFile) {
|
||||
System.out.println("Indexing file " + javaSourceFile.getName());
|
||||
// Parse the Java source file
|
||||
com.github.javaparser.ast.CompilationUnit cu = StaticJavaParser.parse(javaSourceFile);
|
||||
|
||||
// Get all lines of the file
|
||||
List<String> lines = Files.readAllLines(javaSourceFile.toPath());
|
||||
|
||||
// Get the package name
|
||||
String packageName = cu.getPackageDeclaration().map(pd -> pd.getNameAsString()).orElse("");
|
||||
|
||||
// Iterate over each class in the file
|
||||
for (com.github.javaparser.ast.body.ClassOrInterfaceDeclaration cid : cu.findAll(com.github.javaparser.ast.body.ClassOrInterfaceDeclaration.class)) {
|
||||
// Iterate over each method in the class
|
||||
for (Statement md : cid.findAll(Statement.class)) {
|
||||
// Iterate over each line in the method
|
||||
for (int i = md.getBegin().get().line; i <= md.getEnd().get().line; i++) {
|
||||
// Get the line of code
|
||||
String line = lines.get(i - 1);
|
||||
// Create a SourceCodeLine object for the line using the builder pattern
|
||||
SourceCodeLine sourceCodeLine = SourceCodeLine.builder()
|
||||
.line(line.stripLeading().stripTrailing())
|
||||
.lineNumber(i)
|
||||
.fileName(javaSourceFile.getAbsolutePath())
|
||||
.className(cid.getNameAsString())
|
||||
.packageName(packageName)
|
||||
.build();
|
||||
|
||||
// Add the SourceCodeLine object to the index
|
||||
index.put(sourceCodeLine.getClassName(), i, sourceCodeLine);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static void main(String...args) throws IOException {
|
||||
if(args.length < 1) {
|
||||
throw new IllegalArgumentException("Please provide the path to the deeplearning4j root directory");
|
||||
}
|
||||
File nd4jApiRootDir = new File(args[0]);
|
||||
SourceCodeIndexer sourceCodeIndexer = new SourceCodeIndexer(nd4jApiRootDir,new File("oplog.db").getAbsolutePath());
|
||||
ObjectMapper objectMapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
|
||||
objectMapper.writeValue(new FileWriter("index.json"), sourceCodeIndexer);
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
package org.nd4j.interceptor.parser;
|
||||
|
||||
import org.nd4j.shade.guava.collect.HashBasedTable;
|
||||
import org.nd4j.shade.guava.collect.Table;
|
||||
import org.nd4j.shade.jackson.core.JsonParser;
|
||||
import org.nd4j.shade.jackson.databind.DeserializationContext;
|
||||
import org.nd4j.shade.jackson.databind.JsonDeserializer;
|
||||
import org.nd4j.shade.jackson.databind.JsonNode;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
|
||||
public class SourceCodeIndexerDeserializer extends JsonDeserializer<SourceCodeIndexer> {
|
||||
|
||||
@Override
|
||||
public SourceCodeIndexer deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
|
||||
JsonNode node = jsonParser.getCodec().readTree(jsonParser);
|
||||
SourceCodeIndexer sourceCodeIndexer = new SourceCodeIndexer();
|
||||
|
||||
// Load the data as a Map
|
||||
Map<String, Map<Integer, SourceCodeLine>> map = jsonParser.getCodec().treeToValue(node.get("index"), Map.class);
|
||||
|
||||
// Convert the Map to a Table
|
||||
Table<String, Integer, SourceCodeLine> table = HashBasedTable.create();
|
||||
for (Map.Entry<String, Map<Integer, SourceCodeLine>> row : map.entrySet()) {
|
||||
for (Map.Entry<Integer, SourceCodeLine> column : row.getValue().entrySet()) {
|
||||
table.put(row.getKey(), column.getKey(), column.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
sourceCodeIndexer.setIndex(table);
|
||||
return sourceCodeIndexer;
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package org.nd4j.interceptor.parser;
|
||||
|
||||
import org.nd4j.shade.guava.collect.Table;
|
||||
import org.nd4j.shade.jackson.core.JsonGenerator;
|
||||
import org.nd4j.shade.jackson.databind.JsonSerializer;
|
||||
import org.nd4j.shade.jackson.databind.SerializerProvider;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class SourceCodeIndexerSerializer extends JsonSerializer<SourceCodeIndexer> {
|
||||
|
||||
|
||||
@Override
|
||||
public void serialize(SourceCodeIndexer sourceCodeIndexer, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException {
|
||||
jsonGenerator.writeStartObject();
|
||||
|
||||
// Convert the Table to a Map
|
||||
Map<String, Map<Integer, SourceCodeLine>> map = new HashMap<>();
|
||||
for (Table.Cell<String, Integer, SourceCodeLine> cell : sourceCodeIndexer.getIndex().cellSet()) {
|
||||
map.putIfAbsent(cell.getRowKey(), new HashMap<>());
|
||||
map.get(cell.getRowKey()).put(cell.getColumnKey(), cell.getValue());
|
||||
}
|
||||
|
||||
jsonGenerator.writeObjectField("index", map);
|
||||
jsonGenerator.writeEndObject();
|
||||
}
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.interceptor.parser;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Builder
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class SourceCodeLine {
|
||||
|
||||
private String line;
|
||||
private int lineNumber;
|
||||
private String fileName;
|
||||
private String className;
|
||||
private String packageName;
|
||||
public static SourceCodeLine[] from(StackTraceElement[] stackTraceElements) {
|
||||
SourceCodeLine[] sourceCodeLines = new SourceCodeLine[stackTraceElements.length];
|
||||
for (int i = 0; i < stackTraceElements.length; i++) {
|
||||
sourceCodeLines[i] = from(stackTraceElements[i]);
|
||||
}
|
||||
return sourceCodeLines;
|
||||
}
|
||||
|
||||
|
||||
public static SourceCodeLine from(StackTraceElement stackTraceElement) {
|
||||
return SourceCodeLine.builder()
|
||||
.lineNumber(stackTraceElement.getLineNumber())
|
||||
.fileName(stackTraceElement.getFileName())
|
||||
.className(stackTraceElement.getClassName())
|
||||
.packageName(stackTraceElement.getClassName().substring(0, stackTraceElement.getClassName().lastIndexOf(".")))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.interceptor.parser;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class StackTraceMapper {
|
||||
private Map<String, String> mappedStackTraces = new HashMap<>();
|
||||
private Map<String, String> reverseMappedStackTraces = new HashMap<>();
|
||||
|
||||
|
||||
public List<String> getLinesOfCodeFromStackTrace(SourceCodeIndexer indexer, String[] stackTrace) {
|
||||
List<String> linesOfCode = new ArrayList<>();
|
||||
StackTraceElement[] parsedStackTrace = parseStackTrace(stackTrace);
|
||||
|
||||
for (StackTraceElement element : parsedStackTrace) {
|
||||
int lineNumber = element.getLineNumber();
|
||||
String lineOfCode = indexer.getSourceCodeLine(element.getClassName(), lineNumber).getLine();
|
||||
if (lineOfCode != null) {
|
||||
linesOfCode.add(lineOfCode);
|
||||
}
|
||||
}
|
||||
|
||||
return linesOfCode;
|
||||
}
|
||||
|
||||
public void mapStackTraces(String[] stackTrace1, String[] stackTrace2) {
|
||||
List<String> listStackTrace1 = Arrays.asList(stackTrace1);
|
||||
List<String> listStackTrace2 = Arrays.asList(stackTrace2);
|
||||
mapStackTraces(listStackTrace1, listStackTrace2);
|
||||
}
|
||||
|
||||
public void mapStackTraces(List<String> stackTrace1, List<String> stackTrace2) {
|
||||
List<StackTraceElement> parsedStackTrace1 = parseStackTrace(stackTrace1);
|
||||
List<StackTraceElement> parsedStackTrace2 = parseStackTrace(stackTrace2);
|
||||
|
||||
for (StackTraceElement element1 : parsedStackTrace1) {
|
||||
for (StackTraceElement element2 : parsedStackTrace2) {
|
||||
if (element1.getMethodName().equals(element2.getMethodName()) && element1.getLineNumber() == element2.getLineNumber()) {
|
||||
mappedStackTraces.put(element1.toString(), element2.toString());
|
||||
reverseMappedStackTraces.put(element2.toString(), element1.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public String lookupMethod(String method) {
|
||||
return mappedStackTraces.getOrDefault(method, null);
|
||||
}
|
||||
|
||||
public String reverseLookupMethod(String method) {
|
||||
return reverseMappedStackTraces.getOrDefault(method, null);
|
||||
}
|
||||
|
||||
private StackTraceElement[] parseStackTrace(String[] stackTrace) {
|
||||
StackTraceElement[] parsedStackTrace = new StackTraceElement[stackTrace.length];
|
||||
for(int i = 0; i < stackTrace.length; i++) {
|
||||
parsedStackTrace[i] = parseStackTraceLine(stackTrace[i]);
|
||||
}
|
||||
return parsedStackTrace;
|
||||
}
|
||||
|
||||
private List<StackTraceElement> parseStackTrace(List<String> stackTrace) {
|
||||
List<StackTraceElement> parsedStackTrace = new ArrayList<>();
|
||||
|
||||
for (String trace : stackTrace) {
|
||||
StackTraceElement element = parseStackTraceLine(trace);
|
||||
if (element != null) {
|
||||
parsedStackTrace.add(element);
|
||||
}
|
||||
}
|
||||
|
||||
return parsedStackTrace;
|
||||
}
|
||||
|
||||
private StackTraceElement parseStackTraceLine(String stackTraceLine) {
|
||||
String[] parts = stackTraceLine.split("\\.");
|
||||
String className = String.join(".", java.util.Arrays.copyOfRange(parts, 0, parts.length - 1));
|
||||
String methodName = parts[parts.length - 1].split("\\(")[0];
|
||||
String fileName = parts[parts.length - 1].split("\\(")[1].split(":")[0];
|
||||
int lineNumber = Integer.parseInt(parts[parts.length - 1].split("\\(")[1].split(":")[1].replace(")", ""));
|
||||
return new StackTraceElement(className, methodName, fileName, lineNumber);
|
||||
}
|
||||
}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.transformers;
|
||||
|
||||
import net.bytebuddy.agent.builder.AgentBuilder;
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import net.bytebuddy.description.type.TypeDescription;
|
||||
import net.bytebuddy.dynamic.DynamicType;
|
||||
import net.bytebuddy.matcher.ElementMatchers;
|
||||
import net.bytebuddy.utility.JavaModule;
|
||||
import org.nd4j.interceptor.advice.ComputationGraphBackwardAdvice;
|
||||
import org.nd4j.interceptor.advice.ComputationGraphForwardAdvice;
|
||||
|
||||
import java.security.ProtectionDomain;
|
||||
|
||||
public class ComputationGraphTransformer implements AgentBuilder.Transformer {
|
||||
public DynamicType.Builder<?> transform(DynamicType.Builder<?> builder, TypeDescription typeDescription, ClassLoader classLoader, JavaModule javaModule) {
|
||||
builder = builder.visit(Advice.to(ComputationGraphForwardAdvice.class) .on(ElementMatchers.named("output")
|
||||
.or(ElementMatchers.nameContains("ffToLayerActivations"))
|
||||
.or(ElementMatchers.named("outputOfLayerDetached"))));
|
||||
builder = builder.visit(Advice.to(ComputationGraphBackwardAdvice.class).on(ElementMatchers.named("calcBackpropGradients")));
|
||||
return builder;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DynamicType.Builder<?> transform(DynamicType.Builder<?> builder, TypeDescription typeDescription, ClassLoader classLoader, JavaModule javaModule, ProtectionDomain protectionDomain) {
|
||||
return transform(builder, typeDescription, classLoader, javaModule);
|
||||
}
|
||||
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.transformers;
|
||||
|
||||
import net.bytebuddy.agent.builder.AgentBuilder;
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import net.bytebuddy.description.type.TypeDescription;
|
||||
import net.bytebuddy.dynamic.DynamicType;
|
||||
import net.bytebuddy.matcher.ElementMatchers;
|
||||
import net.bytebuddy.utility.JavaModule;
|
||||
import org.nd4j.interceptor.advice.ComputationGraphBackwardAdvice;
|
||||
import org.nd4j.interceptor.advice.ComputationGraphForwardAdvice;
|
||||
import org.nd4j.interceptor.advice.ComputationGraphVertexDoBackwardAdvice;
|
||||
import org.nd4j.interceptor.advice.ComputationGraphVertexDoForwardAdvice;
|
||||
|
||||
import java.security.ProtectionDomain;
|
||||
|
||||
public class ComputationGraphVertexTransformer implements AgentBuilder.Transformer {
|
||||
public DynamicType.Builder<?> transform(DynamicType.Builder<?> builder, TypeDescription typeDescription, ClassLoader classLoader, JavaModule javaModule) {
|
||||
builder = builder.visit(Advice.to(ComputationGraphVertexDoForwardAdvice.class).on(ElementMatchers.named("output")));
|
||||
builder = builder.visit(Advice.to(ComputationGraphVertexDoBackwardAdvice.class).on(ElementMatchers.named("gradientAndScore")));
|
||||
return builder;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DynamicType.Builder<?> transform(DynamicType.Builder<?> builder, TypeDescription typeDescription, ClassLoader classLoader, JavaModule javaModule, ProtectionDomain protectionDomain) {
|
||||
return transform(builder, typeDescription, classLoader, javaModule);
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.transformers;
|
||||
|
||||
import net.bytebuddy.agent.builder.AgentBuilder;
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import net.bytebuddy.description.type.TypeDescription;
|
||||
import net.bytebuddy.dynamic.DynamicType;
|
||||
import net.bytebuddy.matcher.ElementMatchers;
|
||||
import net.bytebuddy.utility.JavaModule;
|
||||
import org.nd4j.interceptor.advice.INDArrayCreationAdvice;
|
||||
import org.nd4j.interceptor.advice.INDArrayUpdateAdvice;
|
||||
import org.nd4j.linalg.api.buffer.DataBuffer;
|
||||
|
||||
import java.security.ProtectionDomain;
|
||||
|
||||
public class INDArrayTransformer implements AgentBuilder.Transformer {
|
||||
public DynamicType.Builder<?> transform(DynamicType.Builder<?> builder, TypeDescription typeDescription, ClassLoader classLoader, JavaModule javaModule) {
|
||||
builder = builder.visit(Advice.to(INDArrayCreationAdvice.class).on(ElementMatchers.isConstructor()));
|
||||
|
||||
builder = builder.visit(Advice.to(INDArrayUpdateAdvice.class).on(ElementMatchers.named("assign")));
|
||||
builder = builder.visit(Advice.to(INDArrayUpdateAdvice.class).on(ElementMatchers.named("putScalar")));
|
||||
builder = builder.visit(Advice.to(INDArrayUpdateAdvice.class).on(ElementMatchers.named("put")));
|
||||
|
||||
return builder;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DynamicType.Builder<?> transform(DynamicType.Builder<?> builder, TypeDescription typeDescription, ClassLoader classLoader, JavaModule javaModule, ProtectionDomain protectionDomain) {
|
||||
return transform(builder, typeDescription, classLoader, javaModule);
|
||||
}
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.transformers;
|
||||
|
||||
import net.bytebuddy.agent.builder.AgentBuilder;
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import net.bytebuddy.description.type.TypeDescription;
|
||||
import net.bytebuddy.dynamic.DynamicType;
|
||||
import net.bytebuddy.matcher.ElementMatchers;
|
||||
import net.bytebuddy.utility.JavaModule;
|
||||
import org.nd4j.interceptor.advice.LayerActivateWithInputAdvice;
|
||||
import org.nd4j.interceptor.advice.LayerBackpropGradientAdvice;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
|
||||
import java.security.ProtectionDomain;
|
||||
|
||||
public class LayerTransformer implements AgentBuilder.Transformer {
|
||||
public DynamicType.Builder<?> transform(DynamicType.Builder<?> builder, TypeDescription typeDescription, ClassLoader classLoader, JavaModule javaModule) {
|
||||
builder = builder.method(ElementMatchers.named("activate")
|
||||
.and(ElementMatchers.takesArgument(0,INDArray.class))
|
||||
.and(ElementMatchers.not(ElementMatchers.isAbstract())))
|
||||
.intercept(Advice.to(LayerActivateWithInputAdvice.class));
|
||||
builder = builder.method(ElementMatchers.named("backpropGradient")
|
||||
.and(ElementMatchers.takesArgument(0,INDArray.class)
|
||||
.and(ElementMatchers.not(ElementMatchers.isAbstract()))))
|
||||
.intercept(Advice.to(LayerBackpropGradientAdvice.class));
|
||||
return builder;
|
||||
|
||||
}
|
||||
@Override
|
||||
public DynamicType.Builder<?> transform(DynamicType.Builder<?> builder, TypeDescription typeDescription, ClassLoader classLoader, JavaModule javaModule, ProtectionDomain protectionDomain) {
|
||||
return transform(builder, typeDescription, classLoader, javaModule);
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.transformers;
|
||||
|
||||
import net.bytebuddy.agent.builder.AgentBuilder;
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import net.bytebuddy.description.type.TypeDescription;
|
||||
import net.bytebuddy.dynamic.DynamicType;
|
||||
import net.bytebuddy.implementation.MethodDelegation;
|
||||
import net.bytebuddy.matcher.ElementMatchers;
|
||||
import net.bytebuddy.utility.JavaModule;
|
||||
import org.nd4j.common.primitives.AtomicBoolean;
|
||||
import org.nd4j.interceptor.advice.ComputationGraphBackwardAdvice;
|
||||
import org.nd4j.interceptor.advice.ComputationGraphForwardAdvice;
|
||||
import org.nd4j.interceptor.advice.MultiLayerNetworkBackwardAdvice;
|
||||
import org.nd4j.interceptor.advice.MultiLayerNetworkForwardAdvice;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.security.ProtectionDomain;
|
||||
|
||||
public class MultiLayerNetworkTransformer implements AgentBuilder.Transformer {
|
||||
|
||||
public DynamicType.Builder<?> transform(DynamicType.Builder<?> builder, TypeDescription typeDescription, ClassLoader classLoader, JavaModule javaModule) {
|
||||
builder = builder.visit(Advice.to(MultiLayerNetworkForwardAdvice.class)
|
||||
.on(ElementMatchers.named("output")
|
||||
.or(ElementMatchers.nameContains("ffToLayerActivations"))
|
||||
.or(ElementMatchers.named("outputOfLayerDetached"))));
|
||||
builder = builder.visit(Advice.to(MultiLayerNetworkBackwardAdvice.class).on(ElementMatchers.named("calcBackpropGradients")));
|
||||
return builder;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DynamicType.Builder<?> transform(DynamicType.Builder<?> builder, TypeDescription typeDescription, ClassLoader classLoader, JavaModule javaModule, ProtectionDomain protectionDomain) {
|
||||
return transform(builder, typeDescription, classLoader, javaModule);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.transformers;
|
||||
|
||||
import net.bytebuddy.agent.builder.AgentBuilder;
|
||||
import net.bytebuddy.asm.Advice;
|
||||
import net.bytebuddy.description.type.TypeDescription;
|
||||
import net.bytebuddy.dynamic.DynamicType;
|
||||
import net.bytebuddy.matcher.ElementMatchers;
|
||||
import net.bytebuddy.utility.JavaModule;
|
||||
import org.nd4j.interceptor.advice.CustomOpAdvice;
|
||||
import org.nd4j.interceptor.advice.OpExecutionerAdvice;
|
||||
import org.nd4j.linalg.api.ops.*;
|
||||
|
||||
import java.security.ProtectionDomain;
|
||||
import java.util.Random;
|
||||
|
||||
public class OpExecutionerTransformer implements AgentBuilder.Transformer {
|
||||
|
||||
public DynamicType.Builder<?> transform(DynamicType.Builder<?> builder, TypeDescription typeDescription, ClassLoader classLoader, JavaModule javaModule) {
|
||||
builder = builder.visit(Advice.to(OpExecutionerAdvice.class).on(ElementMatchers.named("execAndReturn").and(ElementMatchers.takesArguments(1)).and(ElementMatchers.takesArgument(0, TransformOp.class))));
|
||||
builder = builder.visit(Advice.to(OpExecutionerAdvice.class).on(ElementMatchers.named("execAndReturn").and(ElementMatchers.takesArguments(1)).and(ElementMatchers.takesArgument(0, ReduceOp.class))));
|
||||
builder = builder.visit(Advice.to(OpExecutionerAdvice.class).on(ElementMatchers.named("execAndReturn").and(ElementMatchers.takesArguments(1)).and(ElementMatchers.takesArgument(0, IndexAccumulation.class))));
|
||||
builder = builder.visit(Advice.to(OpExecutionerAdvice.class).on(ElementMatchers.named("execAndReturn").and(ElementMatchers.takesArguments(1)).and(ElementMatchers.takesArgument(0, ScalarOp.class))));
|
||||
builder = builder.visit(Advice.to(OpExecutionerAdvice.class).on(ElementMatchers.named("execAndReturn").and(ElementMatchers.takesArguments(1)).and(ElementMatchers.takesArgument(0, BroadcastOp.class))));
|
||||
builder = builder.visit(Advice.to(OpExecutionerAdvice.class).on(ElementMatchers.named("exec").and(ElementMatchers.takesArguments(1)).and(ElementMatchers.takesArgument(0, ReduceOp.class))));
|
||||
builder = builder.visit(Advice.to(OpExecutionerAdvice.class).on(ElementMatchers.named("exec").and(ElementMatchers.takesArguments(1)).and(ElementMatchers.takesArgument(0, BroadcastOp.class))));
|
||||
builder = builder.visit(Advice.to(OpExecutionerAdvice.class).on(ElementMatchers.named("exec").and(ElementMatchers.takesArguments(1)).and(ElementMatchers.takesArgument(0, ScalarOp.class))));
|
||||
builder = builder.visit(Advice.to(OpExecutionerAdvice.class).on(ElementMatchers.named("exec").and(ElementMatchers.takesArguments(1)).and(ElementMatchers.takesArgument(0, IndexAccumulation.class))));
|
||||
builder = builder.visit(Advice.to(OpExecutionerAdvice.class).on(ElementMatchers.named("exec").and(ElementMatchers.takesArguments(1)).and(ElementMatchers.takesArgument(0, MetaOp.class))));
|
||||
builder = builder.visit(Advice.to(OpExecutionerAdvice.class).on(ElementMatchers.named("exec").and(ElementMatchers.takesArguments(1)).and(ElementMatchers.takesArgument(0, GridOp.class))));
|
||||
builder = builder.visit(Advice.to(OpExecutionerAdvice.class).on(ElementMatchers.named("exec").and(ElementMatchers.takesArguments(1)).and(ElementMatchers.takesArgument(0, RandomOp.class))));
|
||||
builder = builder.visit(Advice.to(OpExecutionerAdvice.class).on(ElementMatchers.named("exec").and(ElementMatchers.takesArguments(2)).and(ElementMatchers.takesArgument(0, RandomOp.class)).and(ElementMatchers.takesArgument(1, Random.class))));
|
||||
builder = builder.visit(Advice.to(CustomOpAdvice.class).on(ElementMatchers.named("exec").and(ElementMatchers.takesArguments(2)).and(ElementMatchers.takesArgument(0, CustomOp.class)).and(ElementMatchers.takesArgument(1, OpContext.class))));
|
||||
return builder;
|
||||
|
||||
}
|
||||
|
||||
public DynamicType.Builder<?> transform(DynamicType.Builder<?> builder, TypeDescription typeDescription, ClassLoader classLoader, JavaModule javaModule, ProtectionDomain protectionDomain) {
|
||||
return transform(builder, typeDescription, classLoader, javaModule);
|
||||
}
|
||||
}
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.interceptor.util;
|
||||
|
||||
import org.nd4j.interceptor.InterceptorEnvironment;
|
||||
import org.nd4j.interceptor.data.InterceptorPersistence;
|
||||
import org.nd4j.interceptor.data.OpLogEvent;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.api.ops.CustomOp;
|
||||
import org.nd4j.linalg.api.ops.Op;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.*;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.sql.*;
|
||||
import java.util.*;
|
||||
|
||||
|
||||
public class InterceptorUtils {
|
||||
|
||||
static {
|
||||
try {
|
||||
InterceptorPersistence.bootstrapDatabase(InterceptorEnvironment.CURRENT_FILE_PATH);
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException("Failed to bootstrap database", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public static void logOpExecution(Op op) {
|
||||
if(op.opName().contains("assign")) {
|
||||
return;
|
||||
}
|
||||
if (op.opName().contains("assign")) {
|
||||
StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
|
||||
OpLogEvent opLogEvent = OpLogEvent.builder()
|
||||
.opName(op.opName())
|
||||
.stackTrace(getStackTrace(stackTrace))
|
||||
.firstNonExecutionCodeLine(StackTraceCodeFinder.getFirstLineOfCode(InterceptorEnvironment.SOURCE_CODE_INDEXER_PATH,stackTrace))
|
||||
.inputs(op.y() != null ? convertINDArrayToMap(false, op.x(), op.y()) : convertINDArrayToMap(false, op.x()))
|
||||
.outputs(convertINDArrayToMap(false, op.z()))
|
||||
.build();
|
||||
InterceptorPersistence.addOpLog(opLogEvent);
|
||||
} else {
|
||||
StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
|
||||
OpLogEvent opLogEvent = OpLogEvent.builder()
|
||||
.opName(op.opName())
|
||||
.firstNonExecutionCodeLine(StackTraceCodeFinder.getFirstLineOfCode(InterceptorEnvironment.SOURCE_CODE_INDEXER_PATH,stackTrace))
|
||||
.stackTrace(getStackTrace(stackTrace))
|
||||
.inputs(op.y() != null ? convertINDArrayToMap(true, op.x(), op.y()) : convertINDArrayToMap(true, op.x()))
|
||||
.outputs(convertINDArrayToMap(false, op.z()))
|
||||
.build();
|
||||
InterceptorPersistence.addOpLog(opLogEvent);
|
||||
}
|
||||
}
|
||||
|
||||
private static Map<Integer, String> convertINDArrayToMap(boolean dup, INDArray... arrays) {
|
||||
Map<Integer, String> map = new LinkedHashMap<>();
|
||||
for (int i = 0; i < arrays.length; i++) {
|
||||
INDArray array = arrays[i];
|
||||
String arrayString = array.isView() && dup ? array.dup().toStringFull() : array.toStringFull();
|
||||
map.put(i, arrayString);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
public static void logCustomOpExecution(CustomOp op) {
|
||||
if(op.opName().contains("assign")) {
|
||||
return;
|
||||
}
|
||||
StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
|
||||
OpLogEvent opLogEvent = OpLogEvent.builder()
|
||||
.firstNonExecutionCodeLine(StackTraceCodeFinder.getFirstLineOfCode(InterceptorEnvironment.SOURCE_CODE_INDEXER_PATH,stackTrace))
|
||||
.inputs(convertINDArrayToMap(!op.opName().contains("assign"), op.inputArguments().toArray(new INDArray[0])))
|
||||
.outputs(convertINDArrayToMap(!op.opName().contains("assign"), op.outputArguments().toArray(new INDArray[0])))
|
||||
.opName(op.opName())
|
||||
.stackTrace(getStackTrace())
|
||||
.build();
|
||||
|
||||
InterceptorPersistence.addOpLog(opLogEvent);
|
||||
}
|
||||
|
||||
public static String getStackTrace(StackTraceElement[] stackTrace) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (StackTraceElement element : stackTrace) {
|
||||
sb.append(element.toString()).append(System.lineSeparator());
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
public static String getStackTrace() {
|
||||
return getStackTrace(Thread.currentThread().getStackTrace());
|
||||
}
|
||||
|
||||
}
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.interceptor.util;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.file.*;
|
||||
import java.util.*;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class StackTraceCodeFinder {
|
||||
|
||||
private static final Map<String, Path> filePathCache = new HashMap<>();
|
||||
|
||||
public static String getFirstLineOfCode(String rootDirectory, StackTraceElement[] stackTrace) {
|
||||
if (rootDirectory == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if(!new File(rootDirectory).exists()) {
|
||||
throw new IllegalArgumentException("Root directory does not exist. Unable to scan code path.");
|
||||
}
|
||||
|
||||
Set<String> skipPatterns = new HashSet<>(Arrays.asList(
|
||||
"org\\.nd4j\\.linalg\\.api\\.ops.*",
|
||||
"org\\.nd4j\\.interceptor.*",
|
||||
"org\\.nd4j\\.linalg\\.api\\.ops\\.executioner.*",
|
||||
"java\\.lang\\.*",
|
||||
"org\\.nd4j\\.linalg\\.cpu\\.nativecpu\\.ops.*",
|
||||
"org\\.nd4j\\.linalg\\.jcublas\\.ops\\.executioner.*",
|
||||
"org\\.nd4j\\.linalg\\.factory.*",
|
||||
"org\\.nd4j\\.linalg\\.api\\.ndarray.*",
|
||||
"org\\.nd4j\\.linalg\\.api\\.blas\\.impl.*",
|
||||
"org\\.deeplearning4j\\.nn\\.updater.*"
|
||||
));
|
||||
|
||||
for (StackTraceElement element : stackTrace) {
|
||||
String className = element.getClassName();
|
||||
String packageName = extractPackageName(className);
|
||||
if (shouldSkip(packageName, skipPatterns)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
String line = getLineOfCode(element, rootDirectory);
|
||||
if (line != null) {
|
||||
return line;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public static String extractPackageName(String fullyQualifiedClassName) {
|
||||
int lastDotIndex = fullyQualifiedClassName.lastIndexOf('.');
|
||||
if (lastDotIndex > 0) {
|
||||
return fullyQualifiedClassName.substring(0, lastDotIndex);
|
||||
}
|
||||
return ""; // Default package (no package)
|
||||
}
|
||||
|
||||
|
||||
public static String getLineOfCode(StackTraceElement element, String rootDirectory) {
|
||||
String className = element.getClassName();
|
||||
int lineNumber = element.getLineNumber();
|
||||
|
||||
Path filePath = resolveClassFile(rootDirectory, className);
|
||||
|
||||
if (filePath != null) {
|
||||
try {
|
||||
List<String> lines = Files.readAllLines(filePath);
|
||||
if (lineNumber >= 1 && lineNumber <= lines.size()) {
|
||||
return lines.get(lineNumber - 1);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static boolean shouldSkip(String className, Set<String> skipPatterns) {
|
||||
for (String pattern : skipPatterns) {
|
||||
if (Pattern.matches(pattern, className)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static Path resolveClassFile(String rootDirectory, String fullyQualifiedName) {
|
||||
if (filePathCache.containsKey(fullyQualifiedName)) {
|
||||
return filePathCache.get(fullyQualifiedName);
|
||||
}
|
||||
|
||||
String relativePath = fullyQualifiedName.replace('.', File.separatorChar) + ".java";
|
||||
List<Path> sourceRoots = findSourceRoots(rootDirectory);
|
||||
|
||||
for (Path sourceRoot : sourceRoots) {
|
||||
Path filePath = sourceRoot.resolve(relativePath);
|
||||
if (Files.exists(filePath)) {
|
||||
filePathCache.put(fullyQualifiedName, filePath);
|
||||
return filePath;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static List<Path> findSourceRoots(String rootDirectory) {
|
||||
StackTraceCodeFinderFileVisitor fileVisitor = new StackTraceCodeFinderFileVisitor();
|
||||
try {
|
||||
Files.walkFileTree(Paths.get(rootDirectory), fileVisitor);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return fileVisitor.sourceRoots;
|
||||
}
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.interceptor.util;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.*;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class StackTraceCodeFinderFileVisitor implements FileVisitor<Path> {
|
||||
public List<Path> sourceRoots = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
|
||||
if (dir.endsWith("src/main/java") || dir.endsWith("src/test/java")) {
|
||||
sourceRoots.add(dir);
|
||||
return FileVisitResult.SKIP_SUBTREE;
|
||||
}
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
|
||||
public static void main(String... args) {
|
||||
StackTraceCodeFinderFileVisitor visitor = new StackTraceCodeFinderFileVisitor();
|
||||
try {
|
||||
Files.walkFileTree(Paths.get("/home/agibsonccc/Documents/GitHub/deeplearning4j/"), visitor);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
for (Path p : visitor.sourceRoots) {
|
||||
System.out.println(p);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
Omnihub
|
||||
--------------------------
|
||||
Simple downloading and conversion of pretrained models
|
||||
|
||||
###Setup
|
||||
---------------
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
python setup.py install
|
||||
```
|
||||
|
||||
###Basic Usage
|
||||
See the [unit tests](src/tests/omnihub/test_frameworks.py) for basic usage
|
||||
Simple example:
|
||||
|
||||
```python
|
||||
from omnihub import OnnxModelHub
|
||||
|
||||
keras_model_hub = KerasModelHub()
|
||||
model_path = keras_model_hub.download_model('vgg19/vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5')
|
||||
keras_model_hub.stage_model(model_path, 'vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5')
|
||||
```
|
||||
|
||||
This will download a model using keras applications and put it in:
|
||||
```bash
|
||||
$HOME/.model_hub/keras/vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5
|
||||
```
|
||||
|
||||
The basic idea is that each framework has its "model hub" which knows how interact
|
||||
with and pre process models from different frameworks. The goal is to encapsulate common
|
||||
steps per framework such as freezing/unfreezing, downloading of models.
|
||||
|
||||
|
||||
|
||||
###Background
|
||||
-------------
|
||||
|
||||
An SDK for interacting with various model zoos across different frameworks.
|
||||
Omnihub handles downloading and initializing models from different model zoos
|
||||
handling conversion to standalone files. Various complexities
|
||||
across different frameworks exist for making deployable or finetunable model files.
|
||||
|
||||
Finetuning a model involves usually:
|
||||
1. Unfreezing a model(converting constants to variables)
|
||||
2. Customizing a model (adding a new objective plus other layers on the end)
|
||||
|
||||
Other steps may optionally exist but these are the 2 main ones. Doing this
|
||||
across different frameworks varies in complexity.
|
||||
|
||||
Making a model deployable typically involves:
|
||||
1. freezing a model (convert trainable parameters to frozen constants)
|
||||
2. Optimizing a model (quantizing it, changing the data type, removing extra operations to reduce model size,..)
|
||||
|
||||
These are 2 common workflows that require reusing an existing model file
|
||||
produced by a framework such as tensorflow or pytorch.
|
||||
All of these still come with a fair amount of friction that involves
|
||||
1 off tutorials and copy and paste praying it will work.
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
from transformers import AutoModelForSeq2SeqLM
|
||||
|
||||
from omnihub.frameworks.huggingface import HuggingFaceModelHub
|
||||
from omnihub.frameworks.keras import KerasModelHub
|
||||
from omnihub.frameworks.onnx import OnnxModelHub
|
||||
from omnihub.frameworks.pytorch import PytorchModelHub
|
||||
from omnihub.frameworks.tensorflow import TensorflowModelHub
|
||||
|
||||
keras_model_hub = KerasModelHub()
|
||||
keras_urls = [
|
||||
#'vgg19/vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5',
|
||||
# 'vgg19/vgg19_weights_tf_dim_ordering_tf_kernels.h5',
|
||||
# bug in downloader? Seems to be stalled at the last few bytes, skipping for now
|
||||
#'vgg16/vgg16_weights_tf_dim_ordering_tf_kernels.h5',
|
||||
#'vgg16/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5',
|
||||
'resnet50/notop',
|
||||
'resnet50/top',
|
||||
'resnet101/notop',
|
||||
'resnet101/top',
|
||||
'resnet152/notop',
|
||||
'resnet152/top',
|
||||
'resnet50v2/notop',
|
||||
'resnet50v2/top',
|
||||
'resnet101v2/notop',
|
||||
'resnet101v2/top',
|
||||
'resnet152v2/notop',
|
||||
'resnet152v2/top',
|
||||
'densenet121/notop',
|
||||
'densenet121/top',
|
||||
'densenet169/notop',
|
||||
'densenet169/top',
|
||||
'densenet201/notop',
|
||||
'densenet201/top',
|
||||
'inceptionresnetv2/notop',
|
||||
'inceptionresnetv2/top',
|
||||
'mobilenet/notop',
|
||||
'mobilenet/top',
|
||||
'mobilenetv2/notop',
|
||||
'mobilenetv2/top',
|
||||
#'mobilenetv3/notop',
|
||||
#'mobilenetv3/top',
|
||||
'nasnet/notop',
|
||||
'nasnet/top',
|
||||
'nasnet_mobile/notop',
|
||||
'nasnet_mobile/top',
|
||||
'xception/notop',
|
||||
'xception/top',
|
||||
]
|
||||
for i in range(0,8):
|
||||
keras_urls.append(f'efficientnetb{i}')
|
||||
|
||||
|
||||
|
||||
#for url in keras_urls:
|
||||
# keras_model_hub.download_model(url)
|
||||
onnx_model_hub = OnnxModelHub()
|
||||
onnx_urls = ['vision/body_analysis/age_gender/models/age_googlenet.onnx',
|
||||
'vision/body_analysis/age_gender/models/gender_googlenet.onnx',
|
||||
'vision/body_analysis/age_gender/models/vgg_ilsvrc_16_age_chalearn_iccv2015.onnx',
|
||||
'vision/body_analysis/age_gender/models/vgg_ilsvrc_16_age_imdb_wiki.onnx',
|
||||
'vision/body_analysis/age_gender/models/vgg_ilsvrc_16_gender_imdb_wiki.onnx',
|
||||
'vision/body_analysis/arcface/model/arcfaceresnet100-8.onnx',
|
||||
'vision/body_analysis/emotion_ferplus/model/emotion-ferplus-2.onnx',
|
||||
'vision/body_analysis/emotion_ferplus/model/emotion-ferplus-7.onnx',
|
||||
'vision/body_analysis/emotion_ferplus/model/emotion-ferplus-8.onnx',
|
||||
'vision/body_analysis/ultraface/models/version-RFB-320.onnx',
|
||||
'vision/body_analysis/ultraface/models/version-RFB-640.onnx',
|
||||
'vision/classification/alexnet/model/bvlcalexnet-12-int8.onnx',
|
||||
'vision/classification/alexnet/model/bvlcalexnet-12.onnx',
|
||||
'vision/classification/caffenet/model/caffenet-12-int8.onnx',
|
||||
'vision/classification/caffenet/model/caffenet-12.onnx',
|
||||
'vision/classification/efficientnet-lite4/model/efficientnet-lite4-11.onnx'
|
||||
]
|
||||
#for url in onnx_urls:
|
||||
# onnx_model_hub.download_model(url)
|
||||
tensorflow_model_hub = TensorflowModelHub()
|
||||
tensorflow_urls = ['emilutz/vgg19-block4-conv2-unpooling-decoder/1']
|
||||
for url in tensorflow_urls:
|
||||
tensorflow_model_hub.download_model(url)
|
||||
pytorch_model_hub = PytorchModelHub()
|
||||
pytorch_urls = ['resnet18']
|
||||
for url in pytorch_urls:
|
||||
pytorch_model_hub.download_model(url)
|
||||
huggingface_model_hub = HuggingFaceModelHub()
|
||||
frameworks = ['tensorflow','pytorch']
|
||||
huggingface_urls = [ 'gpt2','bert-base-uncased','t5-base','bert-base-chinese','google/electra-small-discriminator','facebook/wav2vec2-base-960h','facebook/bart-large-cnn']
|
||||
huggingface_urls = ['facebook/bart-large-cnn']
|
||||
for url in huggingface_urls:
|
||||
for framework_name in frameworks:
|
||||
huggingface_model_hub.download_model(url,framework_name=framework_name)
|
||||
@@ -0,0 +1,11 @@
|
||||
keras-applications
|
||||
huggingface-hub
|
||||
tensorflow-hub
|
||||
onnx
|
||||
requests
|
||||
pytest
|
||||
torch-model-archiver
|
||||
cchardet
|
||||
tensorflow
|
||||
transformers
|
||||
scikit-learn
|
||||
@@ -0,0 +1,26 @@
|
||||
import setuptools
|
||||
|
||||
with open("README.md", "r", encoding="utf-8") as fh:
|
||||
long_description = fh.read()
|
||||
|
||||
setuptools.setup(
|
||||
name="omnihub",
|
||||
version="0.0.1",
|
||||
author="Adam Gibson (agibsonccc)",
|
||||
author_email="adam@konduit.ai",
|
||||
description="A small example package",
|
||||
long_description=long_description,
|
||||
long_description_content_type="text/markdown",
|
||||
url="https://github.com/eclipse/deeplearning4j/contrib/modelhub",
|
||||
project_urls={
|
||||
"Bug Tracker": "https://github.com/eclipse/deeplearning4j/issues",
|
||||
},
|
||||
classifiers=[
|
||||
"Programming Language :: Python :: 3",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Operating System :: OS Independent",
|
||||
],
|
||||
package_dir={"": "src"},
|
||||
packages=setuptools.find_packages(where="src"),
|
||||
python_requires=">=3.6"
|
||||
)
|
||||
@@ -0,0 +1,96 @@
|
||||
import os
|
||||
|
||||
from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2
|
||||
|
||||
from omnihub.model_hub import omnihub_dir, ModelHub
|
||||
framework_name = 'huggingface'
|
||||
framework_dir = os.path.join(omnihub_dir, framework_name)
|
||||
BASE_URL = 'https://huggingface.co'
|
||||
from transformers import AutoTokenizer, AutoModel, TFAutoModel
|
||||
import tensorflow as tf
|
||||
import torch
|
||||
|
||||
class HuggingFaceModelHub(ModelHub):
|
||||
def __init__(self):
|
||||
"""
|
||||
Note that when downloading models for usage from huggingface the URLs should take a very specific format.
|
||||
Since huggingface spaces uses git LFS it uses branch names. Huggingface spaces defaults to the main branch.
|
||||
Typically, the URL formula is: https://huggingface.co + the repo name followed by resolve/main/file_name
|
||||
This file name should be a path to a raw model file. For specific framework tools, feel free to reuse
|
||||
code in this repository
|
||||
"""
|
||||
super().__init__(framework_name, BASE_URL)
|
||||
|
||||
def resolve_url(self, repo_path, file_name, branch_name='main'):
|
||||
"""
|
||||
Resolve the file name for downloading from huggingface hub.
|
||||
This creates a path using a branch name with a default of main
|
||||
for downloading models from the hub.
|
||||
:param repo_path: repo path to download from. This is usually
|
||||
the namespace after huggingface.co
|
||||
:param file_name: the file name to download, this should be a model file
|
||||
:param branch_name: the branch name (defaults to main)
|
||||
:return: the real url to use for downloading the target file
|
||||
"""
|
||||
return f'{repo_path}/resolve/{branch_name}/{file_name}'
|
||||
|
||||
def _download_tf(self,model_path):
|
||||
output_model = TFAutoModel.from_pretrained(model_path)
|
||||
return output_model
|
||||
def _download_pytorch(self,model_path):
|
||||
output_model = AutoModel.from_pretrained(model_path)
|
||||
dummy_inputs = output_model.dummy_inputs
|
||||
inputs_ordered = []
|
||||
non_main_inputs = []
|
||||
for name, array in dummy_inputs.items():
|
||||
if name == output_model.main_input_name:
|
||||
inputs_ordered.append(array)
|
||||
else:
|
||||
non_main_inputs.append(array)
|
||||
ordred_dummy_inputs = inputs_ordered + non_main_inputs
|
||||
return output_model, tuple(ordred_dummy_inputs)
|
||||
|
||||
def download_model(self, model_path, **kwargs) -> str:
|
||||
"""
|
||||
Download the model for the given path.
|
||||
A framework_name kwarg is required in order to
|
||||
put the model in the proper location.
|
||||
Due to the nature of huggingface repos being multi framework
|
||||
it's up to the user to specify where a file should go.
|
||||
Valid frameworks are:
|
||||
onnx
|
||||
pytorch
|
||||
tensorflow
|
||||
keras
|
||||
:param model_path: the path to the model to download
|
||||
:param kwargs: a kwargs containing framework_name as described above
|
||||
:return: the path to the model
|
||||
"""
|
||||
assert 'framework_name' in kwargs
|
||||
model_name = model_path.split('/')[-1]
|
||||
framework_name = kwargs.pop('framework_name')
|
||||
|
||||
if framework_name == 'keras' or framework_name == 'tensorflow':
|
||||
output_model = TFAutoModel.from_pretrained(model_path)
|
||||
callable = tf.function(output_model.call)
|
||||
concrete_function = callable.get_concrete_function(output_model.dummy_inputs)
|
||||
frozen = convert_variables_to_constants_v2(concrete_function)
|
||||
graph_def = frozen.graph.as_graph_def()
|
||||
tf.io.write_graph(graph_def, os.path.join(omnihub_dir,'tensorflow'), f'{model_name}.pb', as_text=False)
|
||||
else:
|
||||
download_function = kwargs.get('download_function', self._download_pytorch)
|
||||
if 'download_function' in kwargs:
|
||||
kwargs.pop('download_function')
|
||||
output_model,dummy_inputs = download_function(model_path)
|
||||
torch.onnx.export(output_model,
|
||||
dummy_inputs,
|
||||
f'{os.path.join(omnihub_dir,framework_name)}/{model_name}.onnx',
|
||||
export_params=True,
|
||||
do_constant_folding=False,
|
||||
opset_version=13,
|
||||
**kwargs)
|
||||
|
||||
|
||||
|
||||
def stage_model(self, model_path: str, model_name: str):
|
||||
super().stage_model(model_path, model_name)
|
||||
@@ -0,0 +1,111 @@
|
||||
import os
|
||||
|
||||
from tensorflow.python.keras.applications.mobilenet_v2 import MobileNetV2
|
||||
from tensorflow.python.keras.applications.mobilenet_v3 import MobileNetV3
|
||||
|
||||
from omnihub.model_hub import omnihub_dir, ModelHub
|
||||
from tensorflow.keras.applications.vgg19 import VGG19
|
||||
from tensorflow.keras.applications.vgg16 import VGG16
|
||||
from tensorflow.keras.applications.resnet import ResNet50, ResNet101, ResNet152
|
||||
from tensorflow.keras.applications.resnet_v2 import ResNet50V2, ResNet101V2, ResNet152V2
|
||||
from tensorflow.keras.applications.densenet import DenseNet121, DenseNet169, DenseNet201
|
||||
from tensorflow.keras.applications.inception_resnet_v2 import InceptionResNetV2
|
||||
from tensorflow.keras.applications.efficientnet import EfficientNetB0, EfficientNetB1, EfficientNetB2, EfficientNetB3, \
|
||||
EfficientNetB4, EfficientNetB5, EfficientNetB6, EfficientNetB7
|
||||
from tensorflow.keras.applications.mobilenet import MobileNet
|
||||
from tensorflow.keras.applications.inception_v3 import InceptionV3
|
||||
from tensorflow.keras.applications.nasnet import NASNetLarge, NASNetMobile
|
||||
from tensorflow.keras.applications.xception import Xception
|
||||
framework_name = 'keras'
|
||||
framework_dir = os.path.join(omnihub_dir, framework_name)
|
||||
BASE_URL = 'https://storage.googleapis.com/tensorflow/keras-applications'
|
||||
keras_path = os.path.join(os.path.expanduser('~'), '.keras','models')
|
||||
|
||||
|
||||
class KerasModelHub(ModelHub):
|
||||
def __init__(self):
|
||||
super().__init__(framework_name, BASE_URL)
|
||||
|
||||
def download_model(self, model_path, **kwargs) -> str:
|
||||
"""
|
||||
Download the model and return the model path on the file system
|
||||
:param model_path: the model path for the URL
|
||||
:param kwargs: various kwargs for customizing the underlying behavior
|
||||
:return: the local file path
|
||||
"""
|
||||
model_path = self.download_for_url(model_path,**kwargs)
|
||||
return model_path
|
||||
|
||||
def stage_model(self, model_path: str, model_name: str):
|
||||
super().stage_model(model_path, model_name)
|
||||
|
||||
def download_for_url(self, path: str,**kwargs):
|
||||
"""
|
||||
Download the file at the given URL
|
||||
:param path: the path to download
|
||||
:param kwargs: various kwargs for customizing the underlying behavior of
|
||||
the model download and setup
|
||||
:return: the absolute path to the model
|
||||
"""
|
||||
path_split = path.split('/')
|
||||
type = path_split[0]
|
||||
weights_file = path_split[1]
|
||||
include_top = 'no_top' in weights_file
|
||||
if type == 'vgg19':
|
||||
ret = VGG19(include_top=include_top, **kwargs)
|
||||
elif type == 'vgg16':
|
||||
ret = VGG16(include_top=include_top, **kwargs)
|
||||
elif type == 'resnet50':
|
||||
ret = ResNet50(include_top=include_top, **kwargs)
|
||||
elif type == 'resnet101':
|
||||
ret = ResNet101(include_top=include_top, **kwargs)
|
||||
elif type == 'resnet152':
|
||||
ret = ResNet152(include_top=include_top, **kwargs)
|
||||
elif type == 'resnet50v2':
|
||||
ret = ResNet50V2(include_top=include_top, **kwargs)
|
||||
elif type == 'resnet101v2':
|
||||
ret = ResNet101V2(include_top=include_top, **kwargs)
|
||||
elif type == 'resnet152v2':
|
||||
ret = ResNet152V2(include_top=include_top, **kwargs)
|
||||
elif type == 'densenet121':
|
||||
ret = DenseNet121(include_top=include_top)
|
||||
elif type == 'densenet169':
|
||||
ret = DenseNet169(include_top=include_top, **kwargs)
|
||||
elif type == 'densenet201':
|
||||
ret = DenseNet201(include_top=include_top, **kwargs)
|
||||
elif type == 'inceptionresnetv2':
|
||||
ret = InceptionResNetV2(include_top=include_top, **kwargs)
|
||||
elif type == 'efficientnetb0':
|
||||
ret = EfficientNetB0(include_top=include_top, **kwargs)
|
||||
elif type == 'efficientnetb1':
|
||||
ret = EfficientNetB1(include_top=include_top, **kwargs)
|
||||
elif type == 'efficientnetb2':
|
||||
ret = EfficientNetB2(include_top=include_top, **kwargs)
|
||||
elif type == 'efficientnetb3':
|
||||
ret = EfficientNetB3(include_top=include_top, **kwargs)
|
||||
elif type == 'efficientnetb4':
|
||||
ret = EfficientNetB4(include_top=include_top, **kwargs)
|
||||
elif type == 'efficientnetb5':
|
||||
ret = EfficientNetB5(include_top=include_top, **kwargs)
|
||||
elif type == 'efficientnetb6':
|
||||
ret = EfficientNetB6(include_top=include_top, **kwargs)
|
||||
elif type == 'efficientnetb7':
|
||||
efficient_net = EfficientNetB7(include_top=include_top, **kwargs)
|
||||
elif type == 'mobilenet':
|
||||
ret = MobileNet(include_top=include_top, **kwargs)
|
||||
elif type == 'mobilenetv2':
|
||||
ret = MobileNetV2(include_top=include_top)
|
||||
# MobileNetV3() missing 2 required positional arguments: 'stack_fn' and 'last_point_ch'
|
||||
#elif type == 'mobilenetv3':
|
||||
# mobile_net = MobileNetV3(include_top=include_top, **kwargs)
|
||||
elif type == 'inceptionv3':
|
||||
ret = InceptionV3(include_top=include_top, **kwargs)
|
||||
elif type == 'nasnet':
|
||||
ret = NASNetLarge(include_top=include_top, **kwargs)
|
||||
elif type == 'nasnet_mobile':
|
||||
ret = NASNetMobile(include_top=include_top, **kwargs)
|
||||
elif type == 'xception':
|
||||
ret = Xception(include_top=include_top, **kwargs)
|
||||
model_path = os.path.join(keras_path, weights_file)
|
||||
ret.save(model_path)
|
||||
return model_path
|
||||
@@ -0,0 +1,11 @@
|
||||
import os
|
||||
from omnihub.model_hub import omnihub_dir, ModelHub
|
||||
|
||||
framework_name = 'onnx'
|
||||
framework_dir = os.path.join(omnihub_dir, framework_name)
|
||||
BASE_URL = 'https://media.githubusercontent.com/media/onnx/models/master'
|
||||
|
||||
|
||||
class OnnxModelHub(ModelHub):
|
||||
def __init__(self):
|
||||
super().__init__(framework_name, BASE_URL)
|
||||
@@ -0,0 +1,84 @@
|
||||
import os
|
||||
from omnihub.model_hub import omnihub_dir, ModelHub
|
||||
import torch
|
||||
from torchvision import models
|
||||
import numpy as np
|
||||
|
||||
framework_name = 'pytorch'
|
||||
framework_dir = os.path.join(omnihub_dir, framework_name)
|
||||
BASE_URL = 'https://s3.amazonaws.com/pytorch/models'
|
||||
|
||||
# models with default 224 x 224 height,width
|
||||
MODEL_224_DEFAULTS = ['resnet18', 'vgg16', 'shufflenet_v2_x1_0', 'resnext50_32x4d', 'wide_resnet50_2', 'mnasnet1_0']
|
||||
# models with default 256 x 256 height,width
|
||||
MODEL_256_DEFAULTS = ['alexnet', 'squeezenet1_0', 'densenet161', 'googlenet', 'inception_v3', 'fasterrcnn', 'ssd',
|
||||
'retinanet', 'maskrcnn', 'keypointrcnn']
|
||||
# models in pytorch's model.detection
|
||||
detection_models = ['fasterrcnn', 'ssd', 'retinanet', 'maskrcnn', 'keypointrcnn', 'retinanet']
|
||||
# misc defaults and base dictionary for storing heights,widths
|
||||
MODEL_DEFAULTS = {
|
||||
'mobilenet_v2': {
|
||||
'height': 32,
|
||||
'width': 32
|
||||
},
|
||||
'mobilenet_v3_large': {
|
||||
'height': 320,
|
||||
'width': 320
|
||||
},
|
||||
'mobilenet_v3_small': {
|
||||
'height': 320,
|
||||
'width': 320
|
||||
},
|
||||
'retinanet': {
|
||||
'height': 512,
|
||||
'width': 512
|
||||
},
|
||||
|
||||
}
|
||||
|
||||
# efficient_b0 to 7 has height,width default 256 x 256
|
||||
for i in range(0, 8):
|
||||
MODEL_256_DEFAULTS.append(f'efficientnet_b{i}')
|
||||
|
||||
#regnet_x/y_sizes has default height,width 256 x 256
|
||||
regnet_suffix_sizes = ['400mf', '800mf', '1_6gf', '3_2gf', '8gf', '16gf', '32gf']
|
||||
for suffix in ['x', 'y']:
|
||||
for size in regnet_suffix_sizes:
|
||||
MODEL_256_DEFAULTS.append(f'regnet_{suffix}_{size}')
|
||||
|
||||
for model_224 in MODEL_224_DEFAULTS:
|
||||
MODEL_DEFAULTS[model_224] = {
|
||||
'height': 224,
|
||||
'width': 224
|
||||
}
|
||||
|
||||
for model_256 in MODEL_256_DEFAULTS:
|
||||
MODEL_DEFAULTS[model_256] = {
|
||||
'height': 256,
|
||||
'width': 256
|
||||
}
|
||||
|
||||
|
||||
class PytorchModelHub(ModelHub):
|
||||
def __init__(self):
|
||||
super().__init__(framework_name, BASE_URL)
|
||||
|
||||
def download_model(self, model_path, **kwargs) -> str:
|
||||
model = None
|
||||
height = kwargs.get('height', MODEL_DEFAULTS[model_path]['height'])
|
||||
width = kwargs.get('width', MODEL_DEFAULTS[model_path]['width'])
|
||||
x = torch.from_numpy(np.ones((1, 3, height, width), dtype=np.float32))
|
||||
if model_path in detection_models:
|
||||
model = models.detection[model_path](pretrained=True, **kwargs)
|
||||
else:
|
||||
model = models.__dict__[model_path](pretrained=True, **kwargs)
|
||||
torch.onnx.export(model,
|
||||
x,
|
||||
f'{framework_dir}/{model_path}.onnx',
|
||||
export_params=True,
|
||||
do_constant_folding=False,
|
||||
opset_version=13,
|
||||
**kwargs)
|
||||
|
||||
def stage_model(self, model_path: str, model_name: str):
|
||||
super().stage_model(model_path, model_name)
|
||||
@@ -0,0 +1,45 @@
|
||||
import os
|
||||
|
||||
from tensorflow.core.framework.graph_pb2 import GraphDef
|
||||
|
||||
from omnihub.model_hub import omnihub_dir, ModelHub
|
||||
import tarfile
|
||||
from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2
|
||||
import tensorflow as tf
|
||||
import tempfile
|
||||
|
||||
framework_name = 'tensorflow'
|
||||
framework_dir = os.path.join(omnihub_dir, framework_name)
|
||||
BASE_URL = 'https://tfhub.dev'
|
||||
|
||||
|
||||
def convert_saved_model(saved_model_dir) -> GraphDef:
|
||||
"""
|
||||
Convert the saved model (expanded as a directory)
|
||||
to a frozen graph def
|
||||
:param saved_model_dir: the input model directory
|
||||
:return: the loaded graph def with all parameters in the model
|
||||
"""
|
||||
saved_model = tf.saved_model.load(saved_model_dir)
|
||||
graph_def = saved_model.signatures['serving_default']
|
||||
frozen = convert_variables_to_constants_v2(graph_def)
|
||||
return frozen.graph.as_graph_def()
|
||||
|
||||
|
||||
class TensorflowModelHub(ModelHub):
|
||||
def __init__(self):
|
||||
super().__init__(framework_name, BASE_URL)
|
||||
|
||||
def download_model(self, model_path, **kwargs):
|
||||
final_name = model_path.split('/')[-2]
|
||||
model_path = super().download_model(model_path + '?tf-hub-format=compressed')
|
||||
if not tarfile.is_tarfile(model_path):
|
||||
raise Exception(f'Unable to open tar file at path {model_path}')
|
||||
|
||||
mode = kwargs.get('mode', 'r:gz')
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
with tarfile.open(model_path, mode=mode) as downloaded:
|
||||
downloaded.extractall(tmpdir)
|
||||
tf.io.write_graph(convert_saved_model(tmpdir), framework_dir, f'{final_name}.pb', as_text=False)
|
||||
# remove extra tar file
|
||||
os.remove(model_path)
|
||||
@@ -0,0 +1,70 @@
|
||||
import os
|
||||
import shutil
|
||||
from typing import IO
|
||||
|
||||
import requests
|
||||
|
||||
OMNIHUB_HOME = 'OMNIHUB_HOME'
|
||||
if os.environ.__contains__(OMNIHUB_HOME):
|
||||
omnihub_dir = os.environ[OMNIHUB_HOME]
|
||||
else:
|
||||
omnihub_dir = os.path.join(os.path.expanduser('~'), '.omnihub')
|
||||
|
||||
if not os.path.exists(omnihub_dir):
|
||||
os.mkdir(omnihub_dir)
|
||||
|
||||
|
||||
class ModelHub(object):
|
||||
def __init__(self, framework_name: str, base_url: str):
|
||||
self.framework_name = framework_name
|
||||
self.stage_model_dir = os.path.join(omnihub_dir, self.framework_name)
|
||||
if not os.path.exists(self.stage_model_dir):
|
||||
os.mkdir(self.stage_model_dir)
|
||||
self.base_url = base_url
|
||||
|
||||
def _download_file(self, url: str,**kwargs):
|
||||
local_filename = os.path.join(self.stage_model_dir, url.split('/')[-1])
|
||||
# NOTE the stream=True parameter below
|
||||
with requests.get(url, stream=True) as r:
|
||||
r.raise_for_status()
|
||||
with open(local_filename, 'wb') as f:
|
||||
for chunk in r.iter_content(chunk_size=8192):
|
||||
# If you have chunk encoded response uncomment if
|
||||
# and set chunk_size parameter to None.
|
||||
# if chunk:
|
||||
f.write(chunk)
|
||||
return local_filename
|
||||
|
||||
def download_model(self, model_path,**kwargs) -> str:
|
||||
"""
|
||||
Meant to be overridden by sub classes.
|
||||
Handles downloading a model with the target URL
|
||||
at the path specified.
|
||||
:param model_path: the path to the model from the base URL of the web service
|
||||
:return: the path to the original model
|
||||
"""
|
||||
model_path = self._download_file(f'{self.base_url}/{model_path}')
|
||||
return model_path
|
||||
|
||||
def stage_model(self, model_path: str, model_name: str):
|
||||
"""
|
||||
Copy the model from its original path to the target
|
||||
directory under self.stage_model_dir
|
||||
:param model_path: the original path to the model downloaded
|
||||
by the underlying framework
|
||||
:param model_name: the name of the model file to save as
|
||||
:return:
|
||||
"""
|
||||
shutil.copy(model_path, os.path.join(self.stage_model_dir, model_name))
|
||||
|
||||
def stage_model_stream(self, model_path: IO, model_name: str):
|
||||
"""
|
||||
Copy the model from its original path to the target
|
||||
directory under self.stage_model_dir
|
||||
:param model_path: the original path to the model downloaded
|
||||
by the underlying framework
|
||||
:param model_name: the name of the model file to save as
|
||||
:return:
|
||||
"""
|
||||
with open(os.path.join(self.stage_model_dir, model_name), 'wb+') as f:
|
||||
shutil.copyfileobj(model_path, f)
|
||||
@@ -0,0 +1,38 @@
|
||||
from omnihub.frameworks.huggingface import HuggingFaceModelHub
|
||||
from omnihub.frameworks.keras import KerasModelHub
|
||||
from omnihub.frameworks.onnx import OnnxModelHub
|
||||
from omnihub.frameworks.pytorch import PytorchModelHub
|
||||
from omnihub.frameworks.tensorflow import TensorflowModelHub
|
||||
import os
|
||||
from omnihub.model_hub import omnihub_dir
|
||||
|
||||
|
||||
def test_keras():
|
||||
keras_model_hub = KerasModelHub()
|
||||
model_path = keras_model_hub.download_model('vgg19/vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5')
|
||||
keras_model_hub.stage_model(model_path, 'vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5')
|
||||
assert os.path.exists(os.path.join(omnihub_dir, 'keras', 'vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5'))
|
||||
|
||||
|
||||
def test_onnx():
|
||||
onnx_model_hub = OnnxModelHub()
|
||||
onnx_model_hub.download_model('vision/body_analysis/age_gender/models/age_googlenet.onnx')
|
||||
assert os.path.exists(os.path.join(omnihub_dir, 'onnx', 'age_googlenet.onnx'))
|
||||
|
||||
|
||||
def test_tensorflow():
|
||||
# https://tfhub.dev/emilutz/vgg19-block4-conv2-unpooling-decoder/1?tf-hub-format=compressed
|
||||
tensorflow_model_hub = TensorflowModelHub()
|
||||
tensorflow_model_hub.download_model('emilutz/vgg19-block4-conv2-unpooling-decoder/1')
|
||||
|
||||
|
||||
def test_pytorch():
|
||||
pytorch_model_hub = PytorchModelHub()
|
||||
pytorch_model_hub.download_model('resnet18')
|
||||
assert os.path.exists(os.path.join(omnihub_dir, 'pytorch', 'resnet18.onnx'))
|
||||
|
||||
|
||||
def test_huggingface():
|
||||
huggingface_model_hub = HuggingFaceModelHub()
|
||||
huggingface_model_hub.download_model('gpt2',framework_name='pytorch')
|
||||
#assert os.path.exists(os.path.join(omnihub_dir, 'huggingface', 'tf_model.h5'))
|
||||
@@ -0,0 +1,171 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<!--
|
||||
~ /* ******************************************************************************
|
||||
~ *
|
||||
~ *
|
||||
~ * This program and the accompanying materials are made available under the
|
||||
~ * terms of the Apache License, Version 2.0 which is available at
|
||||
~ * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
~ *
|
||||
~ * See the NOTICE file distributed with this work for additional
|
||||
~ * information regarding copyright ownership.
|
||||
~ * Unless required by applicable law or agreed to in writing, software
|
||||
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
~ * License for the specific language governing permissions and limitations
|
||||
~ * under the License.
|
||||
~ *
|
||||
~ * SPDX-License-Identifier: Apache-2.0
|
||||
~ ******************************************************************************/
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>op-registry-updater</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>OP Registry Updater</name>
|
||||
<description>Standalone tool for updating framework import op registry configurations</description>
|
||||
|
||||
<parent>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>deeplearning4j</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<relativePath>../../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<kotlin.version>1.7.20</kotlin.version>
|
||||
<kotlin.compiler.jvmTarget>11</kotlin.compiler.jvmTarget>
|
||||
<kotlin.compiler.incremental>true</kotlin.compiler.incremental>
|
||||
<main.class>org.eclipse.deeplearning4j.contrib.OpRegistryUpdater</main.class>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<!-- Core framework import dependencies -->
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>samediff-import-api</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>samediff-import-tensorflow</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>samediff-import-onnx</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Kotlin runtime -->
|
||||
<dependency>
|
||||
<groupId>org.jetbrains.kotlin</groupId>
|
||||
<artifactId>kotlin-stdlib-jdk8</artifactId>
|
||||
<version>${kotlin.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Logging -->
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-simple</artifactId>
|
||||
<version>${slf4j.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<sourceDirectory>src/main/kotlin</sourceDirectory>
|
||||
|
||||
<plugins>
|
||||
<!-- Kotlin compilation -->
|
||||
<plugin>
|
||||
<groupId>org.jetbrains.kotlin</groupId>
|
||||
<artifactId>kotlin-maven-plugin</artifactId>
|
||||
<version>${kotlin.version}</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>compile</id>
|
||||
<phase>compile</phase>
|
||||
<goals>
|
||||
<goal>compile</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<jvmTarget>${kotlin.compiler.jvmTarget}</jvmTarget>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<!-- Java compilation -->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.8.1</version>
|
||||
<configuration>
|
||||
<source>${maven.compiler.source}</source>
|
||||
<target>${maven.compiler.target}</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<!-- Exec plugin for running the main class -->
|
||||
<plugin>
|
||||
<groupId>org.codehaus.mojo</groupId>
|
||||
<artifactId>exec-maven-plugin</artifactId>
|
||||
<version>3.1.0</version>
|
||||
<configuration>
|
||||
<mainClass>${main.class}</mainClass>
|
||||
<cleanupDaemonThreads>false</cleanupDaemonThreads>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<!-- Shade plugin to create executable JAR -->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>3.5.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<transformers>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
|
||||
<mainClass>${main.class}</mainClass>
|
||||
</transformer>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
|
||||
<resource>reference.conf</resource>
|
||||
</transformer>
|
||||
</transformers>
|
||||
<filters>
|
||||
<filter>
|
||||
<artifact>*:*</artifact>
|
||||
<excludes>
|
||||
<exclude>META-INF/*.SF</exclude>
|
||||
<exclude>META-INF/*.DSA</exclude>
|
||||
<exclude>META-INF/*.RSA</exclude>
|
||||
</excludes>
|
||||
</filter>
|
||||
</filters>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,45 @@
|
||||
#include <performance/benchmarking/FullBenchmarkSuit.h>
|
||||
#include <performance/benchmarking/LightBenchmarkSuit.h>
|
||||
#include <performance/benchmarking/BenchmarkSuit.h>
|
||||
#include <performance/benchmarking/FullBenchmarkSuit.h>
|
||||
#include <performance/benchmarking/LightBenchmarkSuit.h>
|
||||
|
||||
const char* runLightBenchmarkSuit(bool printOut) {
|
||||
try {
|
||||
sd::LightBenchmarkSuit suit;
|
||||
auto result = suit.runSuit();
|
||||
|
||||
if (printOut)
|
||||
nd4j_printf("%s\n", result.data());
|
||||
|
||||
auto chars = new char[result.length() + 1];
|
||||
std::memcpy(chars, result.data(), result.length());
|
||||
chars[result.length()] = (char) 0x0;
|
||||
|
||||
return chars;
|
||||
} catch (std::exception &e) {
|
||||
sd::LaunchContext::defaultContext()->errorReference()->setErrorCode(1);
|
||||
sd::LaunchContext::defaultContext()->errorReference()->setErrorMessage(e.what());
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
const char* runFullBenchmarkSuit(bool printOut) {
|
||||
try {
|
||||
sd::FullBenchmarkSuit suit;
|
||||
auto result = suit.runSuit();
|
||||
|
||||
if (printOut)
|
||||
nd4j_printf("%s\n", result.data());
|
||||
|
||||
auto chars = new char[result.length() + 1];
|
||||
std::memcpy(chars, result.data(), result.length());
|
||||
chars[result.length()] = (char) 0x0;
|
||||
|
||||
return chars;
|
||||
} catch (std::exception &e) {
|
||||
sd::LaunchContext::defaultContext()->errorReference()->setErrorCode(1);
|
||||
sd::LaunchContext::defaultContext()->errorReference()->setErrorMessage(e.what());
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
|
||||
#ifndef LIBND4J_BENCHMARKSUIT_H
|
||||
#define LIBND4J_BENCHMARKSUIT_H
|
||||
|
||||
#include <string>
|
||||
#include <system/pointercast.h>
|
||||
#include <system/dll.h>
|
||||
#include <helpers/BenchmarkHelper.h>
|
||||
#include <array/NDArrayFactory.h>
|
||||
|
||||
namespace sd {
|
||||
class SD_LIB_EXPORT BenchmarkSuit {
|
||||
public:
|
||||
BenchmarkSuit() = default;
|
||||
~BenchmarkSuit() = default;
|
||||
|
||||
virtual std::string runSuit() = 0;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
#endif //DEV_TESTS_BENCHMARKSUIT_H
|
||||
@@ -0,0 +1,36 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
|
||||
#ifndef LIBND4J_FULLBENCHMARKSUIT_H
|
||||
#define LIBND4J_FULLBENCHMARKSUIT_H
|
||||
|
||||
#include <performance/benchmarking/BenchmarkSuit.h>
|
||||
|
||||
namespace sd {
|
||||
class FullBenchmarkSuit : public BenchmarkSuit {
|
||||
public:
|
||||
std::string runSuit() override;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
#endif //DEV_TESTS_FULLBENCHMARKSUIT_H
|
||||
@@ -0,0 +1,36 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
|
||||
#ifndef LIBND4J_LIGHTBENCHMARKSUIT_H
|
||||
#define LIBND4J_LIGHTBENCHMARKSUIT_H
|
||||
|
||||
#include <performance/benchmarking/BenchmarkSuit.h>
|
||||
|
||||
namespace sd {
|
||||
class LightBenchmarkSuit : public BenchmarkSuit {
|
||||
public:
|
||||
std::string runSuit() override;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
#endif //DEV_TESTS_LIGHTBENCHMARKSUIT_H
|
||||
@@ -0,0 +1,3 @@
|
||||
|
||||
SD_LIB_EXPORT const char* runLightBenchmarkSuit(bool printOut);
|
||||
SD_LIB_EXPORT const char* runFullBenchmarkSuit(bool printOut);
|
||||
@@ -0,0 +1,22 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <performance/benchmarking/BenchmarkSuit.h>
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,642 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
#include "performance/benchmarking/LightBenchmarkSuit.h"
|
||||
|
||||
#ifdef RELEASE_BUILD
|
||||
#define WARMUP 5
|
||||
#define NUM_ITER 100
|
||||
|
||||
#else
|
||||
|
||||
#define WARMUP 5
|
||||
#define NUM_ITER 100
|
||||
|
||||
#endif
|
||||
|
||||
namespace sd {
|
||||
|
||||
template <typename T>
|
||||
static std::string transformBenchmark() {
|
||||
std::string output;
|
||||
output += "transformBenchmark " + DataTypeUtils::asString(DataTypeUtils::fromT<T>());
|
||||
|
||||
BenchmarkHelper helper(WARMUP, NUM_ITER);
|
||||
IntPowerParameters length("length", 2, 8, 20, 4); //2^8, 2^12, 2^16, 2^20 - 4MB
|
||||
BoolParameters inplace("inplace");
|
||||
|
||||
ParametersBatch batch({&length, &inplace});
|
||||
|
||||
auto generator = PARAMETRIC_XZ() {
|
||||
auto arr = NDArrayFactory::create_<T>('c', {p.getIntParam("length")});
|
||||
arr->assign(1.0);
|
||||
x.push_back(arr);
|
||||
if(p.getIntParam("inplace") == 1){
|
||||
z.push_back(arr);
|
||||
} else {
|
||||
z.push_back(NDArrayFactory::create_<T>('c', {p.getIntParam("length")}));
|
||||
}
|
||||
};
|
||||
|
||||
ScalarBenchmark sbRelu(scalar::Ops::RELU, "RELU");
|
||||
sbRelu.setY(NDArrayFactory::create_<T>(0.0));
|
||||
|
||||
TransformBenchmark tbSigmoid(transform::StrictOps::Sigmoid, "sigmoid");
|
||||
//TransformBenchmark tbSoftmax(transform::StrictOps::SoftMax, "softmax");
|
||||
|
||||
output += helper.runOperationSuit(&sbRelu, generator, batch, "RELU");
|
||||
output += helper.runOperationSuit(&tbSigmoid, generator, batch, "Sigmoid");
|
||||
//output += helper.runOperationSuit(&tbSigmoid, generator, batch, "Softmax");
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static std::string scalarBenchmark() {
|
||||
std::string output;
|
||||
output += "scalarBenchmark " + DataTypeUtils::asString(DataTypeUtils::fromT<T>());
|
||||
|
||||
BenchmarkHelper helper(WARMUP, NUM_ITER);
|
||||
|
||||
IntPowerParameters length("length", 2, 8, 20, 4); //2^8, 2^12, 2^16, 2^20
|
||||
BoolParameters inplace("inplace");
|
||||
|
||||
ParametersBatch batch({&length, &inplace});
|
||||
|
||||
auto generator = PARAMETRIC_XZ() {
|
||||
auto arr = NDArrayFactory::create_<T>('c', {p.getIntParam("length")});
|
||||
arr->assign(1.0);
|
||||
x.push_back(arr);
|
||||
if(p.getIntParam("inplace") == 1){
|
||||
z.push_back(arr);
|
||||
} else {
|
||||
z.push_back(NDArrayFactory::create_<T>('c', {p.getIntParam("length")}));
|
||||
}
|
||||
};
|
||||
|
||||
ScalarBenchmark sbAdd(scalar::Ops::Add, "sAdd");
|
||||
ScalarBenchmark sbDiv(scalar::Ops::Divide, "sDiv");
|
||||
ScalarBenchmark sbPow(scalar::Ops::Pow, "sPow");
|
||||
|
||||
|
||||
sbAdd.setY(NDArrayFactory::create_<T>(3.14159265359));
|
||||
sbDiv.setY(NDArrayFactory::create_<T>(3.14159265359));
|
||||
sbPow.setY(NDArrayFactory::create_<T>(3.14159265359));
|
||||
|
||||
|
||||
output += helper.runOperationSuit(&sbAdd, generator, batch, "Scalar Addition - x.add(3.14159265359)");
|
||||
output += helper.runOperationSuit(&sbDiv, generator, batch, "Scalar Division - x.div(3.14159265359)");
|
||||
output += helper.runOperationSuit(&sbPow, generator, batch, "Scalar Power - x.pow(3.14159265359)");
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
static std::string pairwiseBenchmark() {
|
||||
std::string output;
|
||||
output += "pairwiseBenchmark " + DataTypeUtils::asString(DataTypeUtils::fromT<T>());
|
||||
|
||||
BenchmarkHelper helper(WARMUP, NUM_ITER);
|
||||
IntPowerParameters length("length", 2, 8, 20, 4); //2^4 to 2^20 in steps of 4 - 2^4, 2^8, 2^16, 2^20
|
||||
BoolParameters inplace("inplace");
|
||||
|
||||
ParametersBatch batch({&length, &inplace});
|
||||
|
||||
auto generator = PARAMETRIC_XYZ() {
|
||||
auto arr1 = NDArrayFactory::create_<T>('c', {p.getIntParam("length")});
|
||||
auto arr2 = NDArrayFactory::create_<T>('c', {p.getIntParam("length")});
|
||||
x.push_back(arr1);
|
||||
y.push_back(arr2);
|
||||
if(p.getIntParam("inplace") == 1){
|
||||
z.push_back(arr1);
|
||||
} else {
|
||||
z.push_back(NDArrayFactory::create_<T>('c', {p.getIntParam("length")}));
|
||||
}
|
||||
};
|
||||
|
||||
PairwiseBenchmark pb1(pairwise::Ops::Add, "Add");
|
||||
output += helper.runOperationSuit(&pb1, generator, batch, "Pairwise Add");
|
||||
|
||||
PairwiseBenchmark pb2(pairwise::Ops::Divide, "Divide");
|
||||
output += helper.runOperationSuit(&pb2, generator, batch, "Pairwise Divide");
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
static std::string mismatchedOrderAssign() {
|
||||
std::string output;
|
||||
BenchmarkHelper helper(WARMUP, NUM_ITER);
|
||||
|
||||
IntPowerParameters rows("rows", 2, 8, 20, 4); //2^8, 2^12, 2^16, 2^20
|
||||
BoolParameters cf("cf");
|
||||
|
||||
ParametersBatch batch({&rows, &cf});
|
||||
|
||||
auto generator = PARAMETRIC_XZ() {
|
||||
int numElements = 4194304; //2^24
|
||||
int rows = p.getIntParam("rows");
|
||||
int cols = numElements / rows;
|
||||
bool c = p.getIntParam("cf");
|
||||
|
||||
auto arr = NDArrayFactory::create_<float>(c ? 'c' : 'f', {rows, cols});
|
||||
auto arr2 = NDArrayFactory::create_<float>(c ? 'f' : 'c', {rows, cols});
|
||||
x.push_back(arr);
|
||||
z.push_back(arr2);
|
||||
};
|
||||
|
||||
TransformBenchmark tb(transform::AnyOps::Assign, "assign");
|
||||
output += helper.runOperationSuit(&tb, generator, batch, "C->F and F->C Assign F32");
|
||||
|
||||
//Also test: NCHW to NHWC and back
|
||||
BoolParameters nchw("nchw");
|
||||
int mb = 8;
|
||||
int hw = 64;
|
||||
int c = 3;
|
||||
ParametersBatch batch2({&nchw});
|
||||
auto generator2 = PARAMETRIC_XZ() {
|
||||
bool nchw = p.getIntParam("nchw");
|
||||
|
||||
if(nchw) {
|
||||
auto orig = NDArrayFactory::create_<float>('c', {mb, c, hw, hw});
|
||||
orig->permutei({0,2,3,1});
|
||||
x.push_back(orig);
|
||||
z.push_back(NDArrayFactory::create_<float>('c', {mb, hw, hw, c}));
|
||||
} else {
|
||||
auto orig = NDArrayFactory::create_<float>('c', {mb, hw, hw, c});
|
||||
orig->permutei({0,3,1,2});
|
||||
x.push_back(orig);
|
||||
z.push_back(NDArrayFactory::create_<float>('c', {mb, c, hw, hw}));
|
||||
}
|
||||
};
|
||||
|
||||
TransformBenchmark tb2(transform::AnyOps::Assign, "assign_nchw");
|
||||
output += helper.runOperationSuit(&tb2, generator2, batch2, "nchw->nhwc and nhwc->nchw Assign FP32");
|
||||
return output;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static std::string gemmBenchmark() {
|
||||
std::string output;
|
||||
output += "gemm " + DataTypeUtils::asString(DataTypeUtils::fromT<T>());
|
||||
BenchmarkHelper helper(WARMUP, NUM_ITER);
|
||||
|
||||
for (int o = 0; o <= 1; o++) {
|
||||
char resultOrder = (o == 0 ? 'f' : 'c');
|
||||
IntPowerParameters sz("sz", 2, 4, 10, 2); //2^4=16, ..., 2^10=1024 -> 4 elements
|
||||
|
||||
ParametersBatch b({&sz});
|
||||
|
||||
auto generator = PARAMETRIC_XYZ() {
|
||||
auto a = p.getIntParam("sz");
|
||||
auto b = p.getIntParam("sz");
|
||||
auto c = p.getIntParam("sz");
|
||||
std::vector<sd::LongType> shapeA;
|
||||
std::vector<sd::LongType> shapeB;
|
||||
shapeA = {a, b};
|
||||
shapeB = {b, c};
|
||||
auto A = NDArrayFactory::create_<T>('c', shapeA);
|
||||
auto B = NDArrayFactory::create_<T>('c', shapeB);
|
||||
auto C = NDArrayFactory::create_<T>(resultOrder, {a, c});
|
||||
|
||||
x.push_back(A);
|
||||
y.push_back(B);
|
||||
z.push_back(C);
|
||||
};
|
||||
|
||||
std::string n;
|
||||
n += "Gemm - cOrder=";
|
||||
n += resultOrder;
|
||||
|
||||
MatrixBenchmark mb(1.0, 0.0, false, false, n);
|
||||
|
||||
output += helper.runOperationSuit(&mb, generator, b, n.c_str());
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static std::string reduceFullBenchmark() {
|
||||
std::string output;
|
||||
output += "reduceFullBenchmark " + DataTypeUtils::asString(DataTypeUtils::fromT<T>());
|
||||
|
||||
BenchmarkHelper helper(WARMUP, NUM_ITER);
|
||||
|
||||
IntPowerParameters length("length", 2, 8, 20, 4); //2^8, 2^12, 2^16, 2^20
|
||||
|
||||
ParametersBatch batch({&length});
|
||||
|
||||
auto generator = PARAMETRIC_XYZ() {
|
||||
auto arr = NDArrayFactory::create_<T>('c', {p.getIntParam("length")});
|
||||
|
||||
x.push_back(arr);
|
||||
y.push_back(nullptr);
|
||||
z.push_back(NDArrayFactory::create_<T>(0.0f));
|
||||
};
|
||||
|
||||
ReductionBenchmark rbSum(reduce::SameOps::Sum, "sum");
|
||||
ReductionBenchmark rbProd(reduce::SameOps::Prod, "prod");
|
||||
ReductionBenchmark rbMax(reduce::SameOps::Max, "max");
|
||||
|
||||
output += helper.runOperationSuit(&rbSum, (const std::function<void (Parameters &, ResultSet &, ResultSet &, ResultSet &)>)(generator), batch, "Sum - Full Array Reduction");
|
||||
output += helper.runOperationSuit(&rbProd, (const std::function<void (Parameters &, ResultSet &, ResultSet &, ResultSet &)>)(generator), batch, "Product - Full Array Reduction");
|
||||
output += helper.runOperationSuit(&rbMax, (const std::function<void (Parameters &, ResultSet &, ResultSet &, ResultSet &)>)(generator), batch, "Maximum - Full Array Reduction");
|
||||
|
||||
//Index reduction
|
||||
sd::ops::argmax opArgmax;
|
||||
DeclarableBenchmark dbArgmax(opArgmax, "Argmax");
|
||||
auto generator3 = PARAMETRIC_D(){
|
||||
auto ctx = new Context(1);
|
||||
|
||||
ctx->setInputArray(0, NDArrayFactory::create_<T>('c', {p.getIntParam("length")}), true);
|
||||
ctx->setInputArray(1, NDArrayFactory::create_<sd::LongType>((sd::LongType)0), true);
|
||||
ctx->setOutputArray(0, NDArrayFactory::create_<sd::LongType>(0), true);
|
||||
|
||||
return ctx;
|
||||
};
|
||||
output += helper.runOperationSuit(&dbArgmax, generator3, batch, "Argmax Full Array Reduction");
|
||||
return output;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static std::string reduceDimBenchmark(){
|
||||
std::string output;
|
||||
output += "reduceDimBenchmark " + DataTypeUtils::asString(DataTypeUtils::fromT<T>());
|
||||
|
||||
BenchmarkHelper helper(WARMUP, NUM_ITER);
|
||||
|
||||
int length[] = {1024*1024};
|
||||
int pow[] = {10};
|
||||
|
||||
for( int i=0; i<1; i++ ){
|
||||
IntPowerParameters rows("rows", 2, 0, pow[i], 2);
|
||||
BoolParameters dim("dim");
|
||||
|
||||
|
||||
ParametersBatch batch({&rows, &dim});
|
||||
|
||||
auto generator = PARAMETRIC_XYZ() {
|
||||
int rows = p.getIntParam("rows");
|
||||
int cols = length[i] / rows;
|
||||
int dim = p.getIntParam("dim");
|
||||
auto arr = NDArrayFactory::create_<T>('c', {rows, cols});
|
||||
|
||||
|
||||
x.push_back(arr);
|
||||
y.push_back(NDArrayFactory::create_<sd::LongType>(dim));
|
||||
|
||||
NDArray* result;
|
||||
if(dim == 0){
|
||||
result = NDArrayFactory::create_<T>('c', {cols});
|
||||
} else {
|
||||
result = NDArrayFactory::create_<T>('c', {rows});
|
||||
}
|
||||
z.push_back(result);
|
||||
};
|
||||
|
||||
ReductionBenchmark rbSum(reduce::SameOps::Sum, "sum");
|
||||
ReductionBenchmark rbMax(reduce::SameOps::Max, "max");
|
||||
|
||||
std::string s1("Sum Along Dimension - ");
|
||||
s1 += std::to_string(length[i]);
|
||||
std::string s3("Maximum Along Dimension - ");
|
||||
s3 += std::to_string(length[i]);
|
||||
|
||||
output += helper.runOperationSuit(&rbSum, (const std::function<void (Parameters &, ResultSet &, ResultSet &, ResultSet &)>)(generator), batch, s1.c_str());
|
||||
output += helper.runOperationSuit(&rbMax, (const std::function<void (Parameters &, ResultSet &, ResultSet &, ResultSet &)>)(generator), batch, s3.c_str());
|
||||
|
||||
|
||||
|
||||
auto generator3 = PARAMETRIC_D(){
|
||||
auto ctx = new Context(1);
|
||||
int rows = p.getIntParam("rows");
|
||||
int cols = length[i] / rows;
|
||||
int dim = p.getIntParam("dim");
|
||||
auto arr = NDArrayFactory::create_<T>('c', {rows, cols});
|
||||
|
||||
auto dimArg = new sd::LongType[1];
|
||||
dimArg[0] = dim;
|
||||
ctx->setIArguments(dimArg, 1);
|
||||
delete[] dimArg;
|
||||
|
||||
ctx->setInputArray(0, arr, true);
|
||||
|
||||
NDArray* result;
|
||||
if(dim == 0){
|
||||
result = NDArrayFactory::create_<sd::LongType>('c', {cols});
|
||||
} else {
|
||||
result = NDArrayFactory::create_<sd::LongType>('c', {rows});
|
||||
}
|
||||
ctx->setOutputArray(0, result, true);
|
||||
return ctx;
|
||||
};
|
||||
|
||||
std::string s5("Argmax Along Dimension - ");
|
||||
s5 += std::to_string(length[i]);
|
||||
|
||||
sd::ops::argmax opArgmax;
|
||||
DeclarableBenchmark dbArgmax(opArgmax, "Argmax");
|
||||
output += helper.runOperationSuit(&dbArgmax, generator3, batch, s5.c_str());
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static std::string conv2d(){
|
||||
std::string output;
|
||||
output += "conv2d " + DataTypeUtils::asString(DataTypeUtils::fromT<T>());
|
||||
BenchmarkHelper helper(WARMUP, NUM_ITER);
|
||||
|
||||
//Convolution2D op
|
||||
BoolParameters nhwc("nhwc");
|
||||
PredefinedParameters k("k", {2, 3});
|
||||
|
||||
ParametersBatch batch({&nhwc, &k});
|
||||
sd::ops::conv2d conv2d;
|
||||
DeclarableBenchmark benchmark(conv2d, "conv2d");
|
||||
|
||||
int hw = 64;
|
||||
|
||||
auto generator = PARAMETRIC_D() {
|
||||
auto ctx = new Context(1);
|
||||
int n = p.getIntParam("nhwc");
|
||||
int khw = p.getIntParam("k");
|
||||
|
||||
if (n == 0) {
|
||||
auto input = NDArrayFactory::create_<T>('c', {8, 3, hw, hw});
|
||||
auto output = NDArrayFactory::create_<T>('c', {8, 3, hw, hw});
|
||||
ctx->setInputArray(0, input, true);
|
||||
ctx->setOutputArray(0, output, true);
|
||||
} else {
|
||||
auto input = NDArrayFactory::create_<T>('c', {8, hw, hw, 3});
|
||||
auto output = NDArrayFactory::create_<T>('c', {8, hw, hw, 3});
|
||||
ctx->setInputArray(0, input, true);
|
||||
ctx->setOutputArray(0, output, true);
|
||||
}
|
||||
|
||||
auto b = NDArrayFactory::create_<T>('c', {3});
|
||||
auto w = NDArrayFactory::create_<T>('c', {khw, khw, 3, 3}); // [kH, kW, iC, oC] always
|
||||
|
||||
ctx->setInputArray(1, w, true);
|
||||
ctx->setInputArray(2, b, true);
|
||||
|
||||
auto args = new sd::LongType[10];
|
||||
args[0] = args[1] = khw; //Kernel
|
||||
args[2] = args[3] = 1;//Stride
|
||||
args[4] = args[5] = 0; //Pad
|
||||
args[6] = args[7] = 1; //Dilation
|
||||
args[8] = 1; //SAME
|
||||
args[9] = n;//0-nchw, 1=nhwc
|
||||
ctx->setIArguments(args, 10);
|
||||
delete[] args;
|
||||
|
||||
return ctx;
|
||||
};
|
||||
|
||||
output += helper.runOperationSuit(&benchmark, generator, batch, "Conv2d");
|
||||
return output;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static std::string pool2d() {
|
||||
std::string output;
|
||||
output += "pool2d " + DataTypeUtils::asString(DataTypeUtils::fromT<T>());
|
||||
BenchmarkHelper helper(WARMUP, NUM_ITER);
|
||||
|
||||
//Convolution2D op
|
||||
BoolParameters nhwc("nhwc");
|
||||
PredefinedParameters k("k", {2, 3});
|
||||
|
||||
ParametersBatch batch({&nhwc, &k});
|
||||
|
||||
int c = 3;
|
||||
int hw = 64;
|
||||
|
||||
auto generator = PARAMETRIC_D() {
|
||||
auto ctx = new Context(1);
|
||||
int n = p.getIntParam("nhwc");
|
||||
int khw = p.getIntParam("k");
|
||||
|
||||
if (n == 0) {
|
||||
auto input = NDArrayFactory::create_<T>('c', {8, c, hw, hw});
|
||||
auto output = NDArrayFactory::create_<T>('c', {8, c, hw, hw});
|
||||
ctx->setInputArray(0, input, true);
|
||||
ctx->setOutputArray(0, output, true);
|
||||
} else {
|
||||
auto input = NDArrayFactory::create_<T>('c', {8, hw, hw, c});
|
||||
auto output = NDArrayFactory::create_<T>('c', {8, hw, hw, c});
|
||||
ctx->setInputArray(0, input, true);
|
||||
ctx->setOutputArray(0, output, true);
|
||||
}
|
||||
|
||||
auto args = new sd::LongType[11];
|
||||
args[0] = args[1] = khw; //Kernel
|
||||
args[2] = args[3] = 1;//Stride
|
||||
args[4] = args[5] = 0; //Pad
|
||||
args[6] = args[7] = 1; //Dilation
|
||||
args[8] = 1; //SAME
|
||||
args[9] = 0; //Divisor mode - 0 = exclude padding in divisor
|
||||
args[10] = n;//0-nchw, 1=nhwc
|
||||
ctx->setIArguments(args, 11);
|
||||
delete[] args;
|
||||
|
||||
return ctx;
|
||||
};
|
||||
|
||||
sd::ops::avgpool2d avgpool2d;
|
||||
DeclarableBenchmark benchmark1(avgpool2d, "avgpool");
|
||||
output += helper.runOperationSuit(&benchmark1, generator, batch, "Average Pool 2d");
|
||||
|
||||
sd::ops::maxpool2d maxpool2d;
|
||||
DeclarableBenchmark benchmark2(maxpool2d, "maxpool");
|
||||
output += helper.runOperationSuit(&benchmark2, generator, batch, "Max Pool 2d");
|
||||
return output;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static std::string lstmBenchmark() {
|
||||
std::string output;
|
||||
output += "lstm " + DataTypeUtils::asString(DataTypeUtils::fromT<T>());
|
||||
BenchmarkHelper helper(WARMUP, NUM_ITER);
|
||||
|
||||
BoolParameters format("format"); //0=TNS=[seqLen,mb,size]; 1=NST=[mb,size,seqLen]
|
||||
PredefinedParameters mb("mb", {1, 8});
|
||||
int n = 128;
|
||||
|
||||
ParametersBatch batch({&format, &mb});
|
||||
sd::ops::lstmBlock lstmBlock;
|
||||
DeclarableBenchmark benchmark(lstmBlock, "lstm");
|
||||
|
||||
int seqLength = 8;
|
||||
|
||||
auto generator = PARAMETRIC_D() {
|
||||
auto ctx = new Context(1);
|
||||
int f = p.getIntParam("format");
|
||||
int m = p.getIntParam("mb");
|
||||
|
||||
sd::LongType l = 0;
|
||||
ctx->setInputArray(0, NDArrayFactory::create_<sd::LongType>(l), true); //Max TS length (unused)
|
||||
|
||||
|
||||
if (f == 0) {
|
||||
//TNS format
|
||||
ctx->setInputArray(1, NDArrayFactory::create_<T>('c', {seqLength, m, n}), true); //x
|
||||
ctx->setOutputArray(0, NDArrayFactory::create_<T>('c', {seqLength, m, n}), true); //i
|
||||
ctx->setOutputArray(1, NDArrayFactory::create_<T>('c', {seqLength, m, n}), true); //c
|
||||
ctx->setOutputArray(2, NDArrayFactory::create_<T>('c', {seqLength, m, n}), true); //f
|
||||
ctx->setOutputArray(3, NDArrayFactory::create_<T>('c', {seqLength, m, n}), true); //o
|
||||
ctx->setOutputArray(4, NDArrayFactory::create_<T>('c', {seqLength, m, n}), true); //z
|
||||
ctx->setOutputArray(5, NDArrayFactory::create_<T>('c', {seqLength, m, n}), true); //h
|
||||
ctx->setOutputArray(6, NDArrayFactory::create_<T>('c', {seqLength, m, n}), true); //y
|
||||
} else {
|
||||
//NST format
|
||||
ctx->setInputArray(1, NDArrayFactory::create_<T>('f', {m, n, seqLength}), true); //x
|
||||
ctx->setOutputArray(0, NDArrayFactory::create_<T>('f', {m, n, seqLength}), true); //i
|
||||
ctx->setOutputArray(1, NDArrayFactory::create_<T>('f', {m, n, seqLength}), true); //c
|
||||
ctx->setOutputArray(2, NDArrayFactory::create_<T>('f', {m, n, seqLength}), true); //f
|
||||
ctx->setOutputArray(3, NDArrayFactory::create_<T>('f', {m, n, seqLength}), true); //o
|
||||
ctx->setOutputArray(4, NDArrayFactory::create_<T>('f', {m, n, seqLength}), true); //z
|
||||
ctx->setOutputArray(5, NDArrayFactory::create_<T>('f', {m, n, seqLength}), true); //h
|
||||
ctx->setOutputArray(6, NDArrayFactory::create_<T>('f', {m, n, seqLength}), true); //y
|
||||
}
|
||||
|
||||
auto cLast = NDArrayFactory::create_<T>('c', {m, n});
|
||||
auto yLast = NDArrayFactory::create_<T>('c', {m, n});
|
||||
auto W = NDArrayFactory::create_<T>('c', {2 * n, 4 * n});
|
||||
auto Wci = NDArrayFactory::create_<T>('c', {n});
|
||||
auto Wcf = NDArrayFactory::create_<T>('c', {n});
|
||||
auto Wco = NDArrayFactory::create_<T>('c', {n});
|
||||
auto b = NDArrayFactory::create_<T>('c', {4 * n});
|
||||
|
||||
ctx->setInputArray(2, cLast, true);
|
||||
ctx->setInputArray(3, yLast, true);
|
||||
ctx->setInputArray(4, W, true);
|
||||
ctx->setInputArray(5, Wci, true);
|
||||
ctx->setInputArray(6, Wcf, true);
|
||||
ctx->setInputArray(7, Wco, true);
|
||||
ctx->setInputArray(8, b, true);
|
||||
|
||||
auto iargs = new sd::LongType[2];
|
||||
iargs[0] = 0; //No peephole
|
||||
iargs[1] = f;
|
||||
ctx->setIArguments(iargs, 2);
|
||||
delete[] iargs;
|
||||
|
||||
auto targs = new double[2];
|
||||
targs[0] = 1.0; //forget bias
|
||||
targs[1] = 0.0; //cell clipping value
|
||||
ctx->setTArguments(targs, 2);
|
||||
delete[] targs;
|
||||
return ctx;
|
||||
};
|
||||
|
||||
output += helper.runOperationSuit(&benchmark, generator, batch, "LSTMBlock");
|
||||
return output;
|
||||
}
|
||||
|
||||
static std::string broadcast2d() {
|
||||
std::string output;
|
||||
BenchmarkHelper helper(WARMUP, NUM_ITER);
|
||||
|
||||
int rows = 65536;
|
||||
IntPowerParameters cols("cols", 2, 2, 12, 4); //2^2 to 2^12 in steps of 2 - 2^1=2, ..., 2^10=1024
|
||||
BoolParameters axis("axis");
|
||||
BoolParameters inplace("inplace");
|
||||
|
||||
ParametersBatch batch({&cols, &axis, &inplace});
|
||||
|
||||
auto generator = PARAMETRIC_D() {
|
||||
auto a = p.getIntParam("axis");
|
||||
auto arr = NDArrayFactory::create_<float>('c', {rows, p.getIntParam("cols")});
|
||||
|
||||
auto ctx = new Context(1);
|
||||
ctx->setInputArray(0, arr, true);
|
||||
if(a == 0){
|
||||
ctx->setInputArray(1, NDArrayFactory::create_<float>('c', {rows, 1}), true);
|
||||
} else {
|
||||
ctx->setInputArray(1, NDArrayFactory::create_<float>('c', {1, p.getIntParam("cols")}), true);
|
||||
}
|
||||
if (p.getIntParam("inplace") == 1) {
|
||||
ctx->setOutputArray(0, arr);
|
||||
ctx->markInplace(true);
|
||||
} else {
|
||||
ctx->setOutputArray(0, NDArrayFactory::create_<float>('c', {rows, p.getIntParam("cols")}), true);
|
||||
}
|
||||
return ctx;
|
||||
};
|
||||
|
||||
std::string s("add");
|
||||
sd::ops::add op;
|
||||
DeclarableBenchmark benchmark(op, "add");
|
||||
output += helper.runOperationSuit(&benchmark, generator, batch, "Broadcast (Custom) Add - 2d");
|
||||
return output;
|
||||
}
|
||||
|
||||
std::string LightBenchmarkSuit::runSuit() {
|
||||
#ifdef RELEASE_BUILD
|
||||
std::vector<sd::DataType> dtypes({sd::DataType::FLOAT32, sd::DataType::HALF});
|
||||
#else
|
||||
std::vector<sd::DataType> dtypes({sd::DataType::FLOAT32});
|
||||
#endif
|
||||
|
||||
std::string result;
|
||||
|
||||
for (auto t:dtypes) {
|
||||
nd4j_printf("Running LightBenchmarkSuite.transformBenchmark [%s]\n", DataTypeUtils::asString(t).c_str());
|
||||
BUILD_SINGLE_SELECTOR(t, result += transformBenchmark, (), LIBND4J_TYPES);
|
||||
|
||||
nd4j_printf("Running LightBenchmarkSuite.scalarBenchmark [%s]\n", DataTypeUtils::asString(t).c_str());
|
||||
BUILD_SINGLE_SELECTOR(t, result += scalarBenchmark, (), LIBND4J_TYPES);
|
||||
|
||||
nd4j_printf("Running LightBenchmarkSuite.pairwiseBenchmark [%s]\n", DataTypeUtils::asString(t).c_str());
|
||||
BUILD_SINGLE_SELECTOR(t, result += pairwiseBenchmark, (), LIBND4J_TYPES);
|
||||
|
||||
nd4j_printf("Running LightBenchmarkSuite.reduceFullBenchmark [%s]\n", DataTypeUtils::asString(t).c_str());
|
||||
BUILD_SINGLE_SELECTOR(t, result += reduceFullBenchmark, (), LIBND4J_TYPES);
|
||||
|
||||
nd4j_printf("Running LightBenchmarkSuite.reduceDimBenchmark [%s]\n", DataTypeUtils::asString(t).c_str());
|
||||
BUILD_SINGLE_SELECTOR(t, result += reduceDimBenchmark, (), LIBND4J_TYPES);
|
||||
|
||||
nd4j_printf("Running LightBenchmarkSuite.gemmBenchmark [%s]\n", DataTypeUtils::asString(t).c_str());
|
||||
BUILD_SINGLE_SELECTOR(t, result += gemmBenchmark, (), LIBND4J_TYPES);
|
||||
|
||||
nd4j_printf("Running LightBenchmarkSuite.conv2d [%s]\n", DataTypeUtils::asString(t).c_str());
|
||||
BUILD_SINGLE_SELECTOR(t, result += conv2d, (), LIBND4J_TYPES);
|
||||
|
||||
nd4j_printf("Running LightBenchmarkSuite.pool2d [%s]\n", DataTypeUtils::asString(t).c_str());
|
||||
BUILD_SINGLE_SELECTOR(t, result += pool2d, (), LIBND4J_TYPES);
|
||||
|
||||
nd4j_printf("Running LightBenchmarkSuite.lstmBenchmark [%s]\n", DataTypeUtils::asString(t).c_str());
|
||||
BUILD_SINGLE_SELECTOR(t, result += lstmBenchmark, (), LIBND4J_TYPES);
|
||||
|
||||
}
|
||||
|
||||
nd4j_printf("Running LightBenchmarkSuite.broadcast2d\n", "");
|
||||
result += broadcast2d();
|
||||
nd4j_printf("Running LightBenchmarkSuite.mismatchedOrderAssign\n", "");
|
||||
result += mismatchedOrderAssign();
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
# Event Logger Analyzer.
|
||||
|
||||
This proof of concept tool is for analyzing event logs from the EventLogger and UnifiedProfiler.
|
||||
When trying to debug off heap memory leaks it can be hard to understand the allocation patterns.
|
||||
|
||||
The UnifiedProfiler emits logs in both an aggregate mode and singular mode which can
|
||||
either output just runtime and aggregate metrics for workspaces or individual allocation
|
||||
events for further analysis.
|
||||
|
||||
This folder contains classes for helping analyze those logs by converting them to arrow
|
||||
for some basic visualization or import in to a proper analytics database for analysis.
|
||||
@@ -0,0 +1,69 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.example</groupId>
|
||||
<artifactId>unified-profiler-analyzer</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>unified-profiler-analyzer</name>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<dl4j.version>1.0.0-SNAPSHOT</dl4j.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>datavec-arrow</artifactId>
|
||||
<version>${dl4j.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>datavec-api</artifactId>
|
||||
<version>${dl4j.version}</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>datavec-local</artifactId>
|
||||
<version>${dl4j.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.arrow</groupId>
|
||||
<artifactId>arrow-memory-unsafe</artifactId>
|
||||
<version>6.0.1</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.google.code.findbugs</groupId>
|
||||
<artifactId>jsr305</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>nd4j-native</artifactId>
|
||||
<version>${dl4j.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.openjfx</groupId>
|
||||
<artifactId>javafx-maven-plugin</artifactId>
|
||||
<version>0.0.8</version>
|
||||
<configuration>
|
||||
<mainClass>org.eclipse.deeplearning4j.profiler.unifiedprofiler.analysis.UnifiedProfilerLogAnalyzer</mainClass>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
+154
@@ -0,0 +1,154 @@
|
||||
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.eclipse.deeplearning4j.profiler.unifiedprofiler.analysis;
|
||||
|
||||
import javafx.scene.chart.XYChart;
|
||||
import javafx.scene.chart.XYChart.Series;
|
||||
import org.apache.arrow.vector.FieldVector;
|
||||
import org.apache.arrow.vector.UInt8Vector;
|
||||
import org.nd4j.linalg.profiler.data.eventlogger.LogEvent;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Aggregates metrics for global runtime metrics for rendering.
|
||||
* @author Adam Gibson
|
||||
*/
|
||||
public class RuntimeSeries {
|
||||
private Series
|
||||
runtimeFreeMemory,
|
||||
runtimeMaxMemory,
|
||||
javacppAvailablePhysicalBytes,
|
||||
javacppMaxBytes,
|
||||
javacppMaxPhysicalBytes,
|
||||
javacppTotalBytes,
|
||||
javacppPointerCount;
|
||||
|
||||
|
||||
public RuntimeSeries() {
|
||||
runtimeFreeMemory = new Series();
|
||||
runtimeMaxMemory = new Series();
|
||||
javacppAvailablePhysicalBytes = new Series();
|
||||
javacppMaxBytes = new Series();
|
||||
javacppMaxPhysicalBytes = new Series();
|
||||
javacppTotalBytes = new Series();
|
||||
javacppPointerCount = new Series();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Use arrow vectors to record metrics.
|
||||
* @param inputVectors
|
||||
*/
|
||||
public synchronized void record(List<FieldVector> inputVectors) {
|
||||
UInt8Vector eventTimeMsVec = (UInt8Vector) inputVectors.get(0);
|
||||
UInt8Vector runtimeMaxMemoryVec = (UInt8Vector) inputVectors.get(6);
|
||||
UInt8Vector runtimeFreeMemoryVec = (UInt8Vector) inputVectors.get(7);
|
||||
UInt8Vector javacppMaxBytesVec = (UInt8Vector) inputVectors.get(8);
|
||||
UInt8Vector javacppMaxPhysicalBytesVec = (UInt8Vector) inputVectors.get(9);
|
||||
UInt8Vector javacppAvailablePhysicalBytesVec = (UInt8Vector) inputVectors.get(10);
|
||||
UInt8Vector javacppTotalBytesVec = (UInt8Vector) inputVectors.get(11);
|
||||
UInt8Vector javacppPointerCountVec = (UInt8Vector) inputVectors.get(12);
|
||||
for(int i = 0; i < inputVectors.get(0).getValueCount(); i++) {
|
||||
long eventTimeMs = eventTimeMsVec.get(i);
|
||||
long runtimeMaxMemoryVal = runtimeMaxMemoryVec.get(i);
|
||||
long runtimeFreeMemoryVal = runtimeFreeMemoryVec.get(i);
|
||||
long javacppMaxBytesVal = javacppMaxBytesVec.get(i);
|
||||
long javacppMaxPhysicalBytesVal = javacppMaxPhysicalBytesVec.get(i);
|
||||
long javacppAvailablePhysicalBytesVal = javacppAvailablePhysicalBytesVec.get(i);
|
||||
long javacppPointerCountVal = javacppPointerCountVec.get(i);
|
||||
long javacppTotalBytesVal = javacppTotalBytesVec.get(i);
|
||||
runtimeFreeMemory.getData().add(new XYChart.Data<>(runtimeFreeMemoryVal,eventTimeMs));
|
||||
runtimeMaxMemory.getData().add(new XYChart.Data<>(runtimeMaxMemoryVal,eventTimeMs));
|
||||
javacppAvailablePhysicalBytes.getData().add(new XYChart.Data<>(javacppAvailablePhysicalBytesVal,eventTimeMs));
|
||||
javacppMaxBytes.getData().add(new XYChart.Data<>(javacppMaxBytesVal,eventTimeMs));
|
||||
javacppMaxPhysicalBytes.getData().add(new XYChart.Data<>(javacppMaxPhysicalBytesVal,eventTimeMs));
|
||||
javacppTotalBytes.getData().add(new XYChart.Data<>(javacppTotalBytesVal,eventTimeMs));
|
||||
javacppPointerCount.getData().add(new XYChart.Data<>(javacppPointerCountVal,eventTimeMs));
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void record(LogEvent logEvent) {
|
||||
runtimeFreeMemory.getData().add(new XYChart.Data<>(logEvent.getRunTimeMemory().getRuntimeFreeMemory(),logEvent.getEventTimeMs()));
|
||||
runtimeMaxMemory.getData().add(new XYChart.Data<>(logEvent.getRunTimeMemory().getRuntimeMaxMemory(),logEvent.getEventTimeMs()));
|
||||
javacppAvailablePhysicalBytes.getData().add(new XYChart.Data<>(logEvent.getRunTimeMemory().getJavacppAvailablePhysicalBytes(),logEvent.getEventTimeMs()));
|
||||
javacppMaxBytes.getData().add(new XYChart.Data<>(logEvent.getRunTimeMemory().getJavacppMaxBytes(),logEvent.getEventTimeMs()));
|
||||
javacppMaxPhysicalBytes.getData().add(new XYChart.Data<>(logEvent.getRunTimeMemory().getJavacppAvailablePhysicalBytes(),logEvent.getEventTimeMs()));
|
||||
javacppTotalBytes.getData().add(new XYChart.Data<>(logEvent.getRunTimeMemory().getJavacppTotalBytes(),logEvent.getEventTimeMs()));
|
||||
javacppPointerCount.getData().add(new XYChart.Data<>(logEvent.getRunTimeMemory().getJavacppPointerCount(),logEvent.getEventTimeMs()));
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
RuntimeSeries that = (RuntimeSeries) o;
|
||||
return Objects.equals(runtimeFreeMemory, that.runtimeFreeMemory) && Objects.equals(runtimeMaxMemory, that.runtimeMaxMemory) && Objects.equals(javacppAvailablePhysicalBytes, that.javacppAvailablePhysicalBytes) && Objects.equals(javacppMaxBytes, that.javacppMaxBytes) && Objects.equals(javacppMaxPhysicalBytes, that.javacppMaxPhysicalBytes);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(runtimeFreeMemory, runtimeMaxMemory, javacppAvailablePhysicalBytes, javacppMaxBytes, javacppMaxPhysicalBytes);
|
||||
}
|
||||
|
||||
public XYChart.Series getRuntimeFreeMemory() {
|
||||
return runtimeFreeMemory;
|
||||
}
|
||||
|
||||
public void setRuntimeFreeMemory(XYChart.Series runtimeFreeMemory) {
|
||||
this.runtimeFreeMemory = runtimeFreeMemory;
|
||||
}
|
||||
|
||||
public XYChart.Series getRuntimeMaxMemory() {
|
||||
return runtimeMaxMemory;
|
||||
}
|
||||
|
||||
public void setRuntimeMaxMemory(XYChart.Series runtimeMaxMemory) {
|
||||
this.runtimeMaxMemory = runtimeMaxMemory;
|
||||
}
|
||||
|
||||
public XYChart.Series getJavacppAvailablePhysicalBytes() {
|
||||
return javacppAvailablePhysicalBytes;
|
||||
}
|
||||
|
||||
public void setJavacppAvailablePhysicalBytes(XYChart.Series javacppAvailablePhysicalBytes) {
|
||||
this.javacppAvailablePhysicalBytes = javacppAvailablePhysicalBytes;
|
||||
}
|
||||
|
||||
public XYChart.Series getJavacppMaxBytes() {
|
||||
return javacppMaxBytes;
|
||||
}
|
||||
|
||||
public void setJavacppMaxBytes(XYChart.Series javacppMaxBytes) {
|
||||
this.javacppMaxBytes = javacppMaxBytes;
|
||||
}
|
||||
|
||||
public XYChart.Series getJavacppMaxPhysicalBytes() {
|
||||
return javacppMaxPhysicalBytes;
|
||||
}
|
||||
|
||||
public void setJavacppMaxPhysicalBytes(XYChart.Series javacppMaxPhysicalBytes) {
|
||||
this.javacppMaxPhysicalBytes = javacppMaxPhysicalBytes;
|
||||
}
|
||||
}
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.eclipse.deeplearning4j.profiler.unifiedprofiler.analysis;
|
||||
|
||||
import org.datavec.api.records.mapper.RecordMapper;
|
||||
import org.datavec.api.records.reader.impl.csv.CSVRecordReader;
|
||||
import org.datavec.api.split.FileSplit;
|
||||
import org.datavec.api.split.partition.NumberOfRecordsPartitioner;
|
||||
import org.datavec.api.transform.schema.Schema;
|
||||
import org.datavec.arrow.recordreader.ArrowRecordWriter;
|
||||
|
||||
import java.io.*;
|
||||
import java.util.Scanner;
|
||||
|
||||
/**
|
||||
* Convert an aggregate based log to an arrow dataset.
|
||||
*
|
||||
* @author Adam Gibson
|
||||
*/
|
||||
public class UnifiedProfilerAggregateArrowConverter {
|
||||
|
||||
public final static String SPLIT_CSV_DIRECTORY = "split-csv";
|
||||
|
||||
public static void main(String... args) throws Exception {
|
||||
Schema schema = new Schema.Builder()
|
||||
.addColumnLong("eventTimeMs")
|
||||
.addColumnString("associatedWorkspace")
|
||||
.addColumnLong("workspaceSpilledBytes")
|
||||
.addColumnLong("workspacePinnedBytes")
|
||||
.addColumnLong("workspaceExternalBytes")
|
||||
.addColumnLong("workspaceAllocatedMemory")
|
||||
.addColumnLong("runtimeMaxMemory")
|
||||
.addColumnLong("runtimeFreeMemory")
|
||||
.addColumnLong("javacppMaxBytes")
|
||||
.addColumnLong("javacppMaxPhysicalBytes")
|
||||
.addColumnLong("javacppAvailablePhysicalBytes")
|
||||
.addColumnLong("javacppTotalBytes")
|
||||
.addColumnLong("javacppPointerCount")
|
||||
.build();
|
||||
|
||||
|
||||
|
||||
File logFile = new File(args[0]);
|
||||
|
||||
splitLogFile(logFile,10000);
|
||||
CSVRecordReader recordReader = new CSVRecordReader();
|
||||
|
||||
File[] inputFiles = new File(SPLIT_CSV_DIRECTORY).listFiles();
|
||||
for(File inputFile : inputFiles) {
|
||||
FileSplit inputCSv = new FileSplit(inputFile);
|
||||
ArrowRecordWriter arrowRecordWriter = new ArrowRecordWriter(schema);
|
||||
File outputFile = new File("arrow-output/",inputFile.getName() + ".arrow");
|
||||
outputFile.createNewFile();
|
||||
FileSplit outputFileSplit = new FileSplit(outputFile);
|
||||
|
||||
RecordMapper recordMapper = RecordMapper.builder()
|
||||
.recordReader(recordReader)
|
||||
.recordWriter(arrowRecordWriter)
|
||||
.inputUrl(inputCSv)
|
||||
.batchSize(10000)
|
||||
.partitioner(new NumberOfRecordsPartitioner())
|
||||
.outputUrl(outputFileSplit)
|
||||
.callInitRecordReader(true)
|
||||
.callInitRecordWriter(true)
|
||||
.build();
|
||||
recordMapper.copy();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
private static void splitLogFile(File logFile, double numLinesPerFile) throws IOException {
|
||||
Scanner scanner = new Scanner(logFile);
|
||||
int count = 0;
|
||||
while (scanner.hasNextLine()) {
|
||||
scanner.nextLine();
|
||||
count++;
|
||||
}
|
||||
System.out.println("Lines in the file: " + count); // Displays no. of lines in the input file.
|
||||
|
||||
double temp = (count / numLinesPerFile);
|
||||
int temp1 = (int) temp;
|
||||
int nof = 0;
|
||||
if (temp1 == temp) {
|
||||
nof = temp1;
|
||||
} else {
|
||||
nof = temp1 + 1;
|
||||
}
|
||||
System.out.println("No. of files to be generated :" + nof); // Displays no. of files to be generated.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------------
|
||||
|
||||
// Actual splitting of file into smaller files
|
||||
|
||||
FileInputStream fstream = new FileInputStream(logFile);
|
||||
DataInputStream in = new DataInputStream(fstream);
|
||||
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(in));
|
||||
String strLine;
|
||||
File inputDirectory = new File(SPLIT_CSV_DIRECTORY);
|
||||
if(!inputDirectory.exists())
|
||||
inputDirectory.mkdirs();
|
||||
for (int j = 1; j <= nof; j++) {
|
||||
FileWriter fstream1 = new FileWriter(SPLIT_CSV_DIRECTORY + File.separator + "event-log-" + j + ".csv"); // Destination File Location
|
||||
BufferedWriter out = new BufferedWriter(fstream1);
|
||||
for (int i = 1; i <= numLinesPerFile; i++) {
|
||||
strLine = br.readLine();
|
||||
if (strLine != null) {
|
||||
out.write(strLine);
|
||||
if (i != numLinesPerFile) {
|
||||
out.newLine();
|
||||
}
|
||||
}
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
|
||||
in.close();
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+190
@@ -0,0 +1,190 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.eclipse.deeplearning4j.profiler.unifiedprofiler.analysis;
|
||||
|
||||
import javafx.application.Application;
|
||||
import javafx.embed.swing.SwingFXUtils;
|
||||
import javafx.scene.Scene;
|
||||
import javafx.scene.chart.LineChart;
|
||||
import javafx.scene.chart.NumberAxis;
|
||||
import javafx.scene.chart.XYChart;
|
||||
import javafx.scene.image.WritableImage;
|
||||
import javafx.scene.layout.VBox;
|
||||
import javafx.stage.Stage;
|
||||
import javafx.util.StringConverter;
|
||||
import org.apache.arrow.vector.FieldVector;
|
||||
import org.datavec.api.records.reader.impl.transform.TransformProcessRecordReader;
|
||||
import org.datavec.api.split.FileSplit;
|
||||
import org.datavec.api.transform.TransformProcess;
|
||||
import org.datavec.api.transform.schema.Schema;
|
||||
import org.datavec.api.writable.Writable;
|
||||
import org.datavec.arrow.recordreader.ArrowRecordReader;
|
||||
import org.datavec.arrow.recordreader.ArrowWritableRecordBatch;
|
||||
import org.nd4j.common.primitives.Counter;
|
||||
import org.nd4j.common.primitives.CounterMap;
|
||||
import org.nd4j.linalg.api.buffer.DataType;
|
||||
import org.nd4j.linalg.profiler.data.eventlogger.EventType;
|
||||
import org.nd4j.linalg.profiler.data.eventlogger.ObjectAllocationType;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDate;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class UnifiedProfilerAggregateLogAnalyzer extends Application {
|
||||
|
||||
private Counter<EventType> eventTypes = new Counter<>();
|
||||
private Counter<ObjectAllocationType> objectAllocationTypes = new Counter<>();
|
||||
private CounterMap<DataType,EventType> eventTypesByDataType = new CounterMap<>();
|
||||
private static File inputFile;
|
||||
private Stage stage;
|
||||
private static ZoneId defaultZoneId = ZoneId.systemDefault();
|
||||
private static //EEE MMM dd HH:mm:ss zzz
|
||||
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss zzz",Locale.ENGLISH);
|
||||
public UnifiedProfilerAggregateLogAnalyzer(File inputFile, Stage stage) {
|
||||
this.inputFile = inputFile;
|
||||
this.stage = stage;
|
||||
}
|
||||
|
||||
public UnifiedProfilerAggregateLogAnalyzer() {
|
||||
}
|
||||
|
||||
public void analyze() throws Exception {
|
||||
Map<String,WorkspaceSeries> workspaceMemory = new ConcurrentHashMap<>();
|
||||
RuntimeSeries runtimeSeries = new RuntimeSeries();
|
||||
NumberAxis timeAxis = new NumberAxis();
|
||||
timeAxis.setLabel("Time (in ms)");
|
||||
timeAxis.setTickLabelFormatter(new StringConverter<Number>() {
|
||||
@Override
|
||||
public String toString(Number object) {
|
||||
return new Date(object.longValue()).toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Number fromString(String string) {
|
||||
LocalDate date = LocalDate.parse(string, DateTimeFormatter.ISO_DATE);
|
||||
Date date2 = Date.from(date.atStartOfDay(defaultZoneId).toInstant());
|
||||
return date2.getTime();
|
||||
|
||||
}
|
||||
});
|
||||
|
||||
//get the workspace names
|
||||
Schema schema = new Schema.Builder()
|
||||
.addColumnLong("eventTimeMs")
|
||||
.addColumnString("associatedWorkspace")
|
||||
.addColumnLong("workspaceSpilledBytes")
|
||||
.addColumnLong("workspacePinnedBytes")
|
||||
.addColumnLong("workspaceExternalBytes")
|
||||
.addColumnLong("workspaceAllocatedMemory")
|
||||
.addColumnLong("runtimeMaxMemory")
|
||||
.addColumnLong("runtimeFreeMemory")
|
||||
.addColumnLong("javacppMaxBytes")
|
||||
.addColumnLong("javacppMaxPhysicalBytes")
|
||||
.addColumnLong("javacppAvailablePhysicalBytes")
|
||||
.addColumnLong("javacppTotalBytes")
|
||||
.addColumnLong("javacppPointerCount")
|
||||
.build();
|
||||
|
||||
|
||||
ArrowRecordReader arrowRecordReader = new ArrowRecordReader();
|
||||
arrowRecordReader.initialize(new FileSplit(new File("arrow-output")));
|
||||
|
||||
TransformProcess transformProcess = new TransformProcess.Builder(schema)
|
||||
.removeColumns(schema.getColumnNames().stream()
|
||||
.filter(input -> input.equals("associatedWorkspace"))
|
||||
.collect(Collectors.toList()))
|
||||
.build();
|
||||
|
||||
TransformProcessRecordReader transformProcessRecordReader = new TransformProcessRecordReader(arrowRecordReader,transformProcess);
|
||||
Set<String> columns = new HashSet<>();
|
||||
while(transformProcessRecordReader.hasNext()) {
|
||||
List<Writable> next = transformProcessRecordReader.next();
|
||||
columns.addAll(next.stream().map(input -> input.toString()).collect(Collectors.toList()));
|
||||
}
|
||||
|
||||
for(String column : columns) {
|
||||
WorkspaceSeries workspaceSeries = new WorkspaceSeries();
|
||||
workspaceMemory.put(column,workspaceSeries);
|
||||
}
|
||||
|
||||
|
||||
arrowRecordReader = new ArrowRecordReader();
|
||||
arrowRecordReader.initialize(new FileSplit(new File("arrow-output")));
|
||||
while(arrowRecordReader.hasNext()) {
|
||||
arrowRecordReader.next();
|
||||
ArrowWritableRecordBatch currentBatch = arrowRecordReader.getCurrentBatch();
|
||||
List<FieldVector> list = currentBatch.getList();
|
||||
runtimeSeries.record(list);
|
||||
}
|
||||
|
||||
|
||||
render(runtimeSeries.getRuntimeMaxMemory(),timeAxis,"Java heap max memory",new File("runtime-max-memory.png"));
|
||||
render(runtimeSeries.getJavacppAvailablePhysicalBytes(),timeAxis,"Javacpp available physical bytes",new File("javacpp-max-physical-bytes.png"));
|
||||
render(runtimeSeries.getJavacppMaxBytes(),timeAxis,"Javacpp maxphysical bytes",new File("javacpp-max-bytes.png"));
|
||||
render(runtimeSeries.getJavacppAvailablePhysicalBytes(),timeAxis,"Javacpp available physical bytes",new File("javacpp-available-physical-bytes.png"));
|
||||
render(runtimeSeries.getRuntimeFreeMemory(),timeAxis,"Java heap free memory",new File("runtime-free-memory.png"));
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
private void render(XYChart.Series<Number,Number> series,NumberAxis timeChart,String label,File outputFile) throws IOException {
|
||||
NumberAxis allocatedChart = new NumberAxis();
|
||||
allocatedChart.setLabel(label);
|
||||
LineChart<Number,Number> allocatedChart2 = new LineChart<>(allocatedChart,timeChart);
|
||||
allocatedChart2.getData().add(series);
|
||||
saveImage(allocatedChart2,outputFile);
|
||||
|
||||
}
|
||||
|
||||
private void saveImage(LineChart lineChart,File file) throws IOException {
|
||||
//https://stackoverflow.com/questions/29721289/how-to-generate-chart-image-using-javafx-chart-api-for-export-without-displying
|
||||
//save image as above
|
||||
VBox vbox = new VBox(lineChart);
|
||||
|
||||
Scene scene = new Scene(vbox, 400, 200);
|
||||
|
||||
stage.setScene(scene);
|
||||
stage.setHeight(300);
|
||||
stage.setWidth(1200);
|
||||
WritableImage image = scene.snapshot(null);
|
||||
ImageIO.write(SwingFXUtils.fromFXImage(image, null),
|
||||
"png", file);
|
||||
|
||||
}
|
||||
|
||||
public static void main(String...args) throws Exception {
|
||||
UnifiedProfilerAggregateLogAnalyzer.inputFile = new File(args[0]);
|
||||
Application.launch(args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start(Stage stage) throws Exception {
|
||||
UnifiedProfilerAggregateLogAnalyzer unifiedProfilerLogAnalyzer = new UnifiedProfilerAggregateLogAnalyzer(inputFile,stage);
|
||||
unifiedProfilerLogAnalyzer.analyze();
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user