chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:47:05 +08:00
commit 4f3b7da785
7394 changed files with 2005594 additions and 0 deletions
+16
View File
@@ -0,0 +1,16 @@
# Libnd4j Op Descriptor Generator
This module contains a few files for generating op descriptors for libnd4j.
An op descriptor contains its number of inputs and outputs as well as the names
in the code of those attributes when they are assigned.
The main class parses a root directory specified by the user containing the libnd4j code base.
In the libnd4j code base, it will scan for op signatures and certain macros found in https://github.com/eclipse/deeplearning4j/blob/master/libnd4j/include/system/op_boilerplate.h
From that, it will automatically generate a set of op descriptors including counts of types of ops as well as names.
The up to date OpDescriptor class can be found [here](src/main/java/org/nd4j/descriptor/OpDescriptor.java)
The main class can be found [here](src/main/java/org/nd4j/descriptor/ParseGen.java)
+3
View File
@@ -0,0 +1,3 @@
#!/bin/bash
mvn clean package -DskipTests
java -cp target/libnd4j-gen-1.0.0-SNAPSHOT-shaded.jar org.nd4j.descriptor.ParseOpFile "$@"
File diff suppressed because it is too large Load Diff
+214
View File
@@ -0,0 +1,214 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ /* ******************************************************************************
~ *
~ *
~ * This program and the accompanying materials are made available under the
~ * terms of the Apache License, Version 2.0 which is available at
~ * https://www.apache.org/licenses/LICENSE-2.0.
~ *
~ * See the NOTICE file distributed with this work for additional
~ * information regarding copyright ownership.
~ * Unless required by applicable law or agreed to in writing, software
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
~ * License for the specific language governing permissions and limitations
~ * under the License.
~ *
~ * SPDX-License-Identifier: Apache-2.0
~ ******************************************************************************/
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>libnd4j-gen</artifactId>
<version>1.0-SNAPSHOT</version>
<name>libnd4j-gen</name>
<parent>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>codegen</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<nd4j.version>1.0.0-SNAPSHOT</nd4j.version>
<maven-shade-plugin.version>3.5.1</maven-shade-plugin.version>
<javaparser.version>3.24.4</javaparser.version>
</properties>
<dependencies>
<dependency>
<groupId>com.codepoetics</groupId>
<artifactId>protonpack</artifactId>
<version>1.16</version>
</dependency>
<dependency>
<groupId>com.github.javaparser</groupId>
<artifactId>javaparser-core-serialization</artifactId>
<version>${javaparser.version}</version>
</dependency>
<dependency>
<groupId>com.github.javaparser</groupId>
<artifactId>javaparser-symbol-solver-core</artifactId>
<version>${javaparser.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
<version>1.9</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.1</version>
</dependency>
<dependency>
<groupId>org.reflections</groupId>
<artifactId>reflections</artifactId>
<version>0.9.10</version>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>protobuf</artifactId>
<version>${nd4j.version}</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.20</version>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>nd4j-native</artifactId>
<version>${nd4j.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>nd4j-api</artifactId>
<version>${nd4j.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.0.0</version>
<executions>
<execution>
<id>add-source</id>
<phase>generate-sources</phase>
<goals><goal>add-source</goal></goals>
<configuration>
<sources>
<source>src/main/stubs</source>
</sources>
</configuration>
</execution>
<execution>
<id>get-cpu-count</id>
<goals>
<goal>cpu-count</goal>
</goals>
<configuration>
<cpuCount>cpu.core.count</cpuCount>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${maven-shade-plugin.version}</version>
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<createDependencyReducedPom>false</createDependencyReducedPom>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>org/datanucleus/**</exclude>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer" />
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<!-- https://kotlinlang.org/docs/reference/using-maven.html -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<executions>
<!-- Replacing default-compile as it is treated specially by maven -->
<execution>
<id>default-compile</id>
<phase>none</phase>
</execution>
<!-- Replacing default-testCompile as it is treated specially by maven -->
<execution>
<id>default-testCompile</id>
<phase>none</phase>
</execution>
<execution>
<id>java-compile</id>
<phase>compile</phase>
<goals> <goal>compile</goal> </goals>
</execution>
<execution>
<id>java-test-compile</id>
<phase>test-compile</phase>
<goals> <goal>testCompile</goal> </goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
@@ -0,0 +1,131 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.nd4j.descriptor;
import lombok.Builder;
import lombok.Data;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* The op descriptor for the libnd4j code base.
* Each op represents a serialized version of
* {@link org.nd4j.linalg.api.ops.DynamicCustomOp}
* that has naming metadata attached.
*
* @author Adam Gibson
*/
@Data
@Builder(toBuilder = true)
public class OpDeclarationDescriptor implements Serializable {
private String name;
private int nIn,nOut,tArgs,iArgs;
private boolean inplaceAble;
private List<String> inArgNames;
private List<String> outArgNames;
private List<String> tArgNames;
private List<String> iArgNames;
private List<String> bArgNames;
private OpDeclarationType opDeclarationType;
@Builder.Default
private Map<String,Boolean> argOptional = new HashMap<>();
public enum OpDeclarationType {
CUSTOM_OP_IMPL,
BOOLEAN_OP_IMPL,
LIST_OP_IMPL,
LOGIC_OP_IMPL,
OP_IMPL,
DIVERGENT_OP_IMPL,
CONFIGURABLE_OP_IMPL,
REDUCTION_OP_IMPL,
BROADCASTABLE_OP_IMPL,
BROADCASTABLE_BOOL_OP_IMPL,
LEGACY_XYZ,
PLATFORM_IMPL,
PLATFORM_TRANSFORM_STRICT_IMPL,
PLATFORM_SCALAR_OP_IMPL,
PLATFORM_CHECK
}
public void validate() {
if(nIn >= 0 && nIn != inArgNames.size() && !isVariableInputSize()) {
System.err.println("In arg names was not equal to number of inputs found for op " + name);
}
if(nOut >= 0 && nOut != outArgNames.size() && !isVariableOutputSize()) {
System.err.println("Output arg names was not equal to number of outputs found for op " + name);
}
if(tArgs >= 0 && tArgs != tArgNames.size() && !isVariableTArgs()) {
System.err.println("T arg names was not equal to number of T found for op " + name);
}
if(iArgs >= 0 && iArgs != iArgNames.size() && !isVariableIntArgs()) {
System.err.println("Integer arg names was not equal to number of integer args found for op " + name);
}
}
/**
* Returns true if there is a variable number
* of integer arguments for an op
* @return
*/
public boolean isVariableIntArgs() {
return iArgs < 0;
}
/**
* Returns true if there is a variable
* number of t arguments for an op
* @return
*/
public boolean isVariableTArgs() {
return tArgs < 0;
}
/**
* Returns true if the number of outputs is variable size
* @return
*/
public boolean isVariableOutputSize() {
return nOut < 0;
}
/**
* Returns true if the number of
* inputs is variable size
* @return
*/
public boolean isVariableInputSize() {
return nIn < 0;
}
}
@@ -0,0 +1,210 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.nd4j.descriptor;
import org.apache.commons.io.FileUtils;
import org.nd4j.common.base.Preconditions;
import org.nd4j.common.primitives.Pair;
import org.nd4j.descriptor.proposal.ArgDescriptorProposal;
import org.nd4j.descriptor.proposal.ArgDescriptorSource;
import org.nd4j.descriptor.proposal.impl.JavaSourceArgDescriptorSource;
import org.nd4j.descriptor.proposal.impl.Libnd4jArgDescriptorSource;
import org.nd4j.descriptor.proposal.impl.ArgDescriptorParserUtils;
import org.nd4j.ir.OpNamespace;
import org.nd4j.shade.protobuf.TextFormat;
import java.io.File;
import java.nio.charset.Charset;
import java.util.*;
import java.util.stream.Collectors;
/**
* Parses the libnd4j code base based on a relative path
* default of ../deeplearning4j/libnd4j
* or a passed in file path.
* It generates a descriptor for each op.
* The file properties can be found at {@link OpDeclarationDescriptor}
*
*
* @author Adam Gibson
*/
public class ParseOpFile {
public static void main(String...args) throws Exception {
String libnd4jPath = args.length > 0 ? args[0] : Libnd4jArgDescriptorSource.DEFAULT_LIBND4J_DIRECTORY;
String outputFilePath = args.length > 1 ? args[1] : ArgDescriptorParserUtils.DEFAULT_OUTPUT_FILE;
File libnd4jRootDir = new File(libnd4jPath);
StringBuilder nd4jApiSourceDir = new StringBuilder();
nd4jApiSourceDir.append("nd4j");
nd4jApiSourceDir.append(File.separator);
nd4jApiSourceDir.append("nd4j-backends");
nd4jApiSourceDir.append(File.separator);
nd4jApiSourceDir.append("nd4j-api-parent");
nd4jApiSourceDir.append(File.separator);
nd4jApiSourceDir.append("nd4j-api");
nd4jApiSourceDir.append(File.separator);
nd4jApiSourceDir.append("src");
nd4jApiSourceDir.append(File.separator);
nd4jApiSourceDir.append("main");
nd4jApiSourceDir.append(File.separator);
nd4jApiSourceDir.append("java");
File nd4jApiRootDir = new File(new File(libnd4jPath).getParent(),nd4jApiSourceDir.toString());
System.out.println("Parsing libnd4j code base at " + libnd4jRootDir.getAbsolutePath() + " and writing to " + outputFilePath);
Libnd4jArgDescriptorSource libnd4jArgDescriptorSource = Libnd4jArgDescriptorSource.builder()
.libnd4jPath(libnd4jPath)
.weight(99999.0)
.build();
JavaSourceArgDescriptorSource javaSourceArgDescriptorSource = JavaSourceArgDescriptorSource.builder()
.nd4jApiRootDir(nd4jApiRootDir)
.weight(1.0)
.build();
Map<String, OpNamespace.OpDescriptor.OpDeclarationType> opTypes = new HashMap<>();
Map<String,List<ArgDescriptorProposal>> proposals = new HashMap<>();
for(ArgDescriptorSource argDescriptorSource : new ArgDescriptorSource[] {libnd4jArgDescriptorSource,javaSourceArgDescriptorSource}) {
Map<String, List<ArgDescriptorProposal>> currProposals = argDescriptorSource.getProposals();
for(Map.Entry<String,List<ArgDescriptorProposal>> entry : currProposals.entrySet()) {
Preconditions.checkState(!entry.getKey().isEmpty());
Set<String> seenNames = new HashSet<>();
if(proposals.containsKey(entry.getKey())) {
List<ArgDescriptorProposal> currProposalsList = proposals.get(entry.getKey());
currProposalsList.addAll(entry.getValue().stream().filter(proposal -> {
Preconditions.checkState(!proposal.getDescriptor().getName().isEmpty());
boolean ret = proposal.getDescriptor().getArgIndex() >= 0 && !seenNames.contains(proposal.getDescriptor().getName());
seenNames.add(proposal.getDescriptor().getName());
return ret;
}).collect(Collectors.toList()));
}
else {
Preconditions.checkState(!entry.getKey().isEmpty());
proposals.put(entry.getKey(),entry.getValue());
}
}
}
javaSourceArgDescriptorSource.getOpTypes().forEach((k,v) -> {
opTypes.put(k, OpNamespace.OpDescriptor.OpDeclarationType.valueOf(v.name()));
});
libnd4jArgDescriptorSource.getOpTypes().forEach((k,v) -> {
opTypes.put(k, OpNamespace.OpDescriptor.OpDeclarationType.valueOf(v.name()));
});
opTypes.putAll(javaSourceArgDescriptorSource.getOpTypes());
opTypes.putAll(libnd4jArgDescriptorSource.getOpTypes());
OpNamespace.OpDescriptorList.Builder listBuilder = OpNamespace.OpDescriptorList.newBuilder();
for(Map.Entry<String,List<ArgDescriptorProposal>> proposal : proposals.entrySet()) {
Preconditions.checkState(!proposal.getKey().isEmpty());
Map<String, List<ArgDescriptorProposal>> collect = proposal.getValue().stream()
.collect(Collectors.groupingBy(input -> input.getDescriptor().getName()));
//merge boolean and int64
collect.entrySet().forEach(entry -> {
ArgDescriptorParserUtils.standardizeTypes(entry.getValue());
});
Map<Pair<Integer, OpNamespace.ArgDescriptor.ArgType>, OpNamespace.ArgDescriptor> rankedProposals = ArgDescriptorParserUtils.
standardizeNames(collect, proposal.getKey());
OpNamespace.OpDescriptor.Builder opDescriptorBuilder = OpNamespace.OpDescriptor.newBuilder()
.setOpDeclarationType(opTypes.get(proposal.getKey()))
.setName(proposal.getKey());
rankedProposals.entrySet().stream().map(input -> input.getValue())
.forEach(argDescriptor -> {
opDescriptorBuilder.addArgDescriptor(argDescriptor);
});
listBuilder.addOpList(opDescriptorBuilder.build());
}
OpNamespace.OpDescriptorList.Builder sortedListBuilder = OpNamespace.OpDescriptorList.newBuilder();
List<OpNamespace.OpDescriptor> sortedDescriptors = new ArrayList<>();
for(int i = 0; i < listBuilder.getOpListCount(); i++) {
OpNamespace.OpDescriptor opList = listBuilder.getOpList(i);
OpNamespace.OpDescriptor.Builder sortedOpBuilder = OpNamespace.OpDescriptor.newBuilder();
Map<OpNamespace.ArgDescriptor.ArgType, List<OpNamespace.ArgDescriptor>> sortedByType = opList.getArgDescriptorList().stream().collect(Collectors.groupingBy(input -> input.getArgType()));
Set<String> namesEncountered = new HashSet<>();
sortedByType.entrySet().forEach(entry -> {
Collections.sort(entry.getValue(),Comparator.comparing(inputArg -> inputArg.getArgIndex()));
for(int j = 0; j < entry.getValue().size(); j++) {
OpNamespace.ArgDescriptor currDescriptor = entry.getValue().get(j);
boolean isArrayArg = false;
String finalName = currDescriptor.getName();
if(currDescriptor.getName().contains("[")) {
isArrayArg = true;
finalName = finalName.replaceAll("\\[.*\\]","").replace("*","");
}
if(currDescriptor.getArgIndex() != j && !namesEncountered.contains(currDescriptor.getName())) {
throw new IllegalStateException("Op name " + opList.getName() + " has incontiguous indices for type " + entry.getKey() + " with descriptor being " +currDescriptor);
} else if(currDescriptor.getArgIndex() != j && namesEncountered.contains(currDescriptor.getName())) {
//skip names we already mapped
System.err.println("Op name " + opList.getName() + " has incontiguous indices for type " + entry.getKey() + " with descriptor being " +currDescriptor + " skipping");
}
OpNamespace.ArgDescriptor.Builder newDescriptor = OpNamespace.ArgDescriptor.newBuilder()
.setName(finalName)
.setArgIndex(currDescriptor.getArgIndex())
.setIsArray(isArrayArg)
.setArgType(currDescriptor.getArgType())
.setConvertBoolToInt(currDescriptor.getConvertBoolToInt());
sortedOpBuilder.addArgDescriptor(newDescriptor.build());
namesEncountered.add(currDescriptor.getName());
}
});
sortedOpBuilder.setOpDeclarationType(opList.getOpDeclarationType());
sortedOpBuilder.setName(opList.getName());
sortedDescriptors.add(sortedOpBuilder.build());
}
//sort alphabetically
Collections.sort(sortedDescriptors,Comparator.comparing(opDescriptor -> opDescriptor.getName()));
//add placeholder as an op to map
sortedDescriptors.add(OpNamespace.OpDescriptor.newBuilder()
.setName("placeholder")
.setOpDeclarationType(OpNamespace.OpDescriptor.OpDeclarationType.LOGIC_OP_IMPL)
.build());
sortedDescriptors.forEach(input -> {
sortedListBuilder.addOpList(input);
});
String write = TextFormat.printToString(sortedListBuilder.build());
FileUtils.writeStringToFile(new File(outputFilePath),write, Charset.defaultCharset());
}
}
@@ -0,0 +1,38 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.nd4j.descriptor.proposal;
import lombok.Builder;
import lombok.Data;
import org.nd4j.ir.OpNamespace;
@Builder
@Data
public class ArgDescriptorProposal {
private OpNamespace.ArgDescriptor descriptor;
private String sourceLine;
private double proposalWeight;
private String sourceOfProposal;
}
@@ -0,0 +1,35 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.nd4j.descriptor.proposal;
import org.nd4j.ir.OpNamespace;
import java.util.List;
import java.util.Map;
public interface ArgDescriptorSource {
Map<String, List<ArgDescriptorProposal>> getProposals();
OpNamespace.OpDescriptor.OpDeclarationType typeFor(String name);
}
@@ -0,0 +1,842 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.nd4j.descriptor.proposal.impl;
import com.github.javaparser.ast.expr.MethodCallExpr;
import com.github.javaparser.resolution.declarations.ResolvedMethodDeclaration;
import com.github.javaparser.resolution.declarations.ResolvedParameterDeclaration;
import lombok.val;
import org.apache.commons.text.similarity.LevenshteinDistance;
import org.nd4j.autodiff.samediff.SDVariable;
import org.nd4j.common.base.Preconditions;
import org.nd4j.common.primitives.*;
import org.nd4j.descriptor.OpDeclarationDescriptor;
import org.nd4j.descriptor.proposal.ArgDescriptorProposal;
import org.nd4j.ir.OpNamespace;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ndarray.INDArray;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class ArgDescriptorParserUtils {
public final static String DEFAULT_OUTPUT_FILE = "op-ir.proto";
public final static Pattern numberPattern = Pattern.compile("\\([\\d]+\\)");
public final static String ARGUMENT_ENDING_PATTERN = "\\([\\w\\d+-\\\\*\\/]+\\);";
public final static String ARGUMENT_PATTERN = "\\([\\w\\d+-\\\\*\\/]+\\)";
public final static String ARRAY_ASSIGNMENT = "\\w+\\[[a-zA-Z]+\\]\\s*=\\s*[A-Z]+_[A-Z]+\\(\\s*[\\d\\w+-\\/\\*\\s]+\\);";
public final static Set<String> bannedMaxIndexOps = new HashSet<String>() {{
add("embedding_lookup");
add("stack");
}};
public final static Set<String> bannedIndexChangeOps = new HashSet<String>() {{
add("gemm");
add("mmul");
add("matmul");
}};
public static final Set<String> cppTypes = new HashSet<String>() {{
add("int");
add("bool");
add("auto");
add("string");
add("float");
add("double");
add("char");
add("class");
add("uint");
}};
public final static Set<String> fieldNameFilters = new HashSet<String>() {{
add("sameDiff");
add("xVertexId");
add("yVertexId");
add("zVertexId");
add("extraArgs");
add("extraArgz");
add("dimensionz");
add("scalarValue");
add("dimensions");
add("jaxis");
add("inPlace");
}};
public final static Set<String> fieldNameFiltersDynamicCustomOps = new HashSet<String>() {{
add("sameDiff");
add("xVertexId");
add("yVertexId");
add("zVertexId");
add("extraArgs");
add("extraArgz");
add("dimensionz");
add("scalarValue");
add("jaxis");
add("inPlace");
add("inplaceCall");
}};
public static Map<String,String> equivalentAttributeNames = new HashMap<String,String>() {{
put("axis","dimensions");
put("dimensions","axis");
put("jaxis","dimensions");
put("dimensions","jaxis");
put("inplaceCall","inPlace");
put("inPlace","inplaceCall");
}};
public static Set<String> dimensionNames = new HashSet<String>() {{
add("jaxis");
add("axis");
add("dimensions");
add("dimensionz");
add("dim");
add("axisVector");
add("axesI");
add("ax");
add("dims");
add("axes");
add("axesVector");
}};
public static Set<String> inputNames = new HashSet<String>() {{
add("input");
add("inputs");
add("i_v");
add("x");
add("in");
add("args");
add("i_v1");
add("first");
add("layerInput");
add("in1");
add("arrays");
}};
public static Set<String> input2Names = new HashSet<String>() {{
add("y");
add("i_v2");
add("second");
add("in2");
}};
public static Set<String> outputNames = new HashSet<String>() {{
add("output");
add("outputs");
add("out");
add("result");
add("z");
add("outputArrays");
}};
public static Set<String> inplaceNames = new HashSet<String>() {{
add("inPlace");
add("inplaceCall");
}};
public static OpNamespace.ArgDescriptor.ArgType argTypeForParam(ResolvedParameterDeclaration parameterDeclaration) {
String type = parameterDeclaration.describeType();
boolean isEnum = false;
try {
isEnum = Class.forName(parameterDeclaration.asParameter().describeType()).isEnum();
} catch(ClassNotFoundException e) {
}
if(type.contains(INDArray.class.getName()) || type.contains(SDVariable.class.getName())) {
if(!outputNames.contains(parameterDeclaration.getName())) {
return OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR;
}
else return OpNamespace.ArgDescriptor.ArgType.OUTPUT_TENSOR;
} else if(type.contains(DataType.class.getName())) {
return OpNamespace.ArgDescriptor.ArgType.DATA_TYPE;
} else if(type.contains(double.class.getName()) || type.contains(float.class.getName()) || type.contains(Float.class.getName()) || type.contains(Double.class.getName())) {
return OpNamespace.ArgDescriptor.ArgType.DOUBLE;
} else if(type.contains(int.class.getName()) || type.contains(long.class.getName()) ||
type.contains(Integer.class.getName()) || type.contains(Long.class.getName()) || isEnum) {
return OpNamespace.ArgDescriptor.ArgType.INT64;
} else if(type.contains(boolean.class.getName()) || type.contains(Boolean.class.getName())) {
return OpNamespace.ArgDescriptor.ArgType.BOOL;
} else {
return OpNamespace.ArgDescriptor.ArgType.UNRECOGNIZED;
}
}
public static boolean paramIsEnum(String paramType) {
try {
return Class.forName(paramType).isEnum();
} catch(ClassNotFoundException e) {
return false;
}
}
public static boolean paramIsEnum(ResolvedParameterDeclaration param) {
return paramIsEnum(param.describeType());
}
public static boolean isValidParam(ResolvedParameterDeclaration param) {
boolean describedClassIsEnum = false;
boolean ret = param.describeType().contains(INDArray.class.getName()) ||
param.describeType().contains(boolean.class.getName()) ||
param.describeType().contains(Boolean.class.getName()) ||
param.describeType().contains(SDVariable.class.getName()) ||
param.describeType().contains(Integer.class.getName()) ||
param.describeType().contains(int.class.getName()) ||
param.describeType().contains(double.class.getName()) ||
param.describeType().contains(Double.class.getName()) ||
param.describeType().contains(float.class.getName()) ||
param.describeType().contains(Float.class.getName()) ||
param.describeType().contains(Long.class.getName()) ||
param.describeType().contains(long.class.getName());
try {
describedClassIsEnum = Class.forName(param.asParameter().describeType()).isEnum();
} catch(ClassNotFoundException e) {
}
return ret || describedClassIsEnum;
}
public static ResolvedMethodDeclaration tryResolve(MethodCallExpr methodCallExpr) {
try {
return methodCallExpr.resolve();
}catch(Exception e) {
}
return null;
}
public static boolean typeNameOrArrayOfTypeNameMatches(String typeName,String...types) {
boolean ret = false;
for(String type : types) {
ret = typeName.equals(type) ||
typeName.equals(type + "...") ||
typeName.equals(type + "[]") || ret;
}
return ret;
}
public static boolean equivalentAttribute(OpNamespace.ArgDescriptor comp1, OpNamespace.ArgDescriptor comp2) {
if(equivalentAttributeNames.containsKey(comp1.getName())) {
return equivalentAttributeNames.get(comp1.getName()).equals(comp2.getName());
}
if(equivalentAttributeNames.containsKey(comp2.getName())) {
return equivalentAttributeNames.get(comp2.getName()).equals(comp1.getName());
}
return false;
}
public static boolean argsListContainsEquivalentAttribute(List<OpNamespace.ArgDescriptor> argDescriptors, OpNamespace.ArgDescriptor to) {
for(OpNamespace.ArgDescriptor argDescriptor : argDescriptors) {
if(argDescriptor.getArgType() == to.getArgType() && equivalentAttribute(argDescriptor,to)) {
return true;
}
}
return false;
}
public static boolean argsListContainsSimilarArg(List<OpNamespace.ArgDescriptor> argDescriptors, OpNamespace.ArgDescriptor to, int threshold) {
for(OpNamespace.ArgDescriptor argDescriptor : argDescriptors) {
if(argDescriptor.getArgType() == to.getArgType() && LevenshteinDistance.getDefaultInstance().apply(argDescriptor.getName().toLowerCase(),to.getName().toLowerCase()) <= threshold) {
return true;
}
}
return false;
}
public static OpNamespace.ArgDescriptor mergeDescriptorsOfSameIndex(OpNamespace.ArgDescriptor one, OpNamespace.ArgDescriptor two) {
if(one.getArgIndex() != two.getArgIndex()) {
throw new IllegalArgumentException("Argument indices for both arg descriptors were not the same. First one was " + one.getArgIndex() + " and second was " + two.getArgIndex());
}
if(one.getArgType() != two.getArgType()) {
throw new IllegalArgumentException("Merging two arg descriptors requires both be the same type. First one was " + one.getArgType().name() + " and second one was " + two.getArgType().name());
}
OpNamespace.ArgDescriptor.Builder newDescriptor = OpNamespace.ArgDescriptor.newBuilder();
//arg indices will be the same
newDescriptor.setArgIndex(one.getArgIndex());
newDescriptor.setArgType(one.getArgType());
if(!isValidIdentifier(one.getName()) && !isValidIdentifier(two.getName())) {
newDescriptor.setName("arg" + newDescriptor.getArgIndex());
} else if(!isValidIdentifier(one.getName())) {
newDescriptor.setName(two.getName());
} else {
newDescriptor.setName(one.getName());
}
return newDescriptor.build();
}
public static boolean isValidIdentifier(String input) {
if(input == null || input.isEmpty())
return false;
for(int i = 0; i < input.length(); i++) {
if(!Character.isJavaIdentifierPart(input.charAt(i)))
return false;
}
if(cppTypes.contains(input))
return false;
return true;
}
public static boolean containsOutputTensor(Collection<ArgDescriptorProposal> proposals) {
for(ArgDescriptorProposal proposal : proposals) {
if(proposal.getDescriptor().getArgType() == OpNamespace.ArgDescriptor.ArgType.OUTPUT_TENSOR) {
return true;
}
}
return false;
}
public static OpNamespace.ArgDescriptor getDescriptorWithName(String name, Collection<ArgDescriptorProposal> proposals) {
for(ArgDescriptorProposal proposal : proposals) {
if(proposal.getDescriptor().getName().equals(name)) {
return proposal.getDescriptor();
}
}
return null;
}
public static int numProposalsWithType(OpNamespace.ArgDescriptor.ArgType argType, Collection<ArgDescriptorProposal> proposals) {
int count = 0;
for(ArgDescriptorProposal proposal : proposals) {
if(proposal.getDescriptor().getArgType() == argType) {
count++;
}
}
return count;
}
public static boolean containsProposalWithDescriptorName(String name, Collection<ArgDescriptorProposal> proposals) {
for(ArgDescriptorProposal proposal : proposals) {
if(proposal.getDescriptor().getName().equals(name)) {
return true;
}
}
return false;
}
public List<ArgDescriptorProposal> updateOpDescriptor(OpNamespace.OpDescriptor opDescriptor, OpDeclarationDescriptor declarationDescriptor, List<String> argsByIIndex, OpNamespace.ArgDescriptor.ArgType int64) {
List<OpNamespace.ArgDescriptor> copyValuesInt = addArgDescriptors(opDescriptor, declarationDescriptor, argsByIIndex, int64);
List<ArgDescriptorProposal> proposals = new ArrayList<>();
return proposals;
}
public static List<OpNamespace.ArgDescriptor> addArgDescriptors(OpNamespace.OpDescriptor opDescriptor, OpDeclarationDescriptor declarationDescriptor, List<String> argsByTIndex, OpNamespace.ArgDescriptor.ArgType argType) {
List<OpNamespace.ArgDescriptor> copyValuesFloat = new ArrayList<>(opDescriptor.getArgDescriptorList());
for(int i = 0; i < argsByTIndex.size(); i++) {
OpNamespace.ArgDescriptor argDescriptor = OpNamespace.ArgDescriptor.newBuilder()
.setArgType(argType)
.setName(argsByTIndex.get(i))
.setArgIndex(i)
//this can happen when there are still missing names from c++
.setArgOptional(declarationDescriptor != null && i <= declarationDescriptor.getTArgs() ? false : true)
.build();
copyValuesFloat.add(argDescriptor);
}
return copyValuesFloat;
}
public static Map<String,Integer> argIndexForCsv(String line) {
Map<String,Integer> ret = new HashMap<>();
String[] lineSplit = line.split(",");
for(int i = 0; i < lineSplit.length; i++) {
ret.put(lineSplit[i],i);
}
return ret;
}
public static Integer extractArgFromJava(String line) {
Matcher matcher = numberPattern.matcher(line);
if(!matcher.find()) {
throw new IllegalArgumentException("No number found for line " + line);
}
return Integer.parseInt(matcher.group());
}
public static Integer extractArgFromCpp(String line,String argType) {
Matcher matcher = Pattern.compile(argType + "\\([\\d]+\\)").matcher(line);
if(!matcher.find()) {
//Generally not resolvable
return -1;
}
if(matcher.groupCount() > 1) {
throw new IllegalArgumentException("Line contains more than 1 index");
}
try {
return Integer.parseInt(matcher.group().replace("(","").replace(")","").replace(argType,""));
} catch(NumberFormatException e) {
e.printStackTrace();
return -1;
}
}
public static List<Field> getAllFields(Class clazz) {
if (clazz == null) {
return Collections.emptyList();
}
List<Field> result = new ArrayList<>(getAllFields(clazz.getSuperclass()));
List<Field> filteredFields = Arrays.stream(clazz.getDeclaredFields())
.filter(f -> Modifier.isPublic(f.getModifiers()) || Modifier.isProtected(f.getModifiers()))
.collect(Collectors.toList());
result.addAll(filteredFields);
return result;
}
public static String removeBracesFromDeclarationMacro(String line, String nameOfMacro) {
line = line.replace(nameOfMacro + "(", "");
line = line.replace(")", "");
line = line.replace("{", "");
line = line.replace(";","");
return line;
}
public static void addNameToList(String line, List<String> list, List<Integer> argIndices, String argType) {
String[] split = line.split(" = ");
String[] arrSplit = split[0].split(" ");
//type + name
String name = arrSplit[arrSplit.length - 1];
Preconditions.checkState(!name.isEmpty());
if(!list.contains(name))
list.add(name);
Integer index = extractArgFromCpp(line,argType);
if(index != null)
argIndices.add(index);
}
public static void addArrayNameToList(String line, List<String> list, List<Integer> argIndices, String argType) {
String[] split = line.split(" = ");
String[] arrSplit = split[0].split(" ");
//type + name
String name = arrSplit[arrSplit.length - 1];
Preconditions.checkState(!name.isEmpty());
if(!list.contains(name))
list.add(name);
//arrays are generally appended to the end
Integer index = - 1;
if(index != null)
argIndices.add(index);
}
public static void standardizeTypes(List<ArgDescriptorProposal> input) {
input.stream().forEach(proposal -> {
//note that if automatic conversion should not happen, set convertBoolToInt to false
if(proposal.getDescriptor().getArgType() == OpNamespace.ArgDescriptor.ArgType.BOOL && proposal.getDescriptor().getConvertBoolToInt()) {
OpNamespace.ArgDescriptor newDescriptor = OpNamespace.ArgDescriptor.newBuilder()
.setArgIndex(proposal.getDescriptor().getArgIndex())
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName(proposal.getDescriptor().getName())
.build();
proposal.setDescriptor(newDescriptor);
}
});
}
public static ArgDescriptorProposal aggregateProposals(List<ArgDescriptorProposal> listOfProposals) {
val descriptorBuilder = OpNamespace.ArgDescriptor.newBuilder();
Counter<Integer> mostLikelyIndex = new Counter<>();
AtomicDouble aggregatedWeight = new AtomicDouble(0.0);
listOfProposals.forEach(proposal -> {
mostLikelyIndex.incrementCount(proposal.getDescriptor().getArgIndex(),1.0);
aggregatedWeight.addAndGet(proposal.getProposalWeight());
descriptorBuilder.setName(proposal.getDescriptor().getName());
descriptorBuilder.setIsArray(proposal.getDescriptor().getIsArray());
descriptorBuilder.setArgType(proposal.getDescriptor().getArgType());
descriptorBuilder.setConvertBoolToInt(proposal.getDescriptor().getConvertBoolToInt());
descriptorBuilder.setIsArray(proposal.getDescriptor().getIsArray());
});
//set the index after computing the most likely index
descriptorBuilder.setArgIndex(mostLikelyIndex.argMax());
return ArgDescriptorProposal
.builder()
.descriptor(descriptorBuilder.build())
.proposalWeight(aggregatedWeight.doubleValue())
.build();
}
public static Map<Pair<Integer, OpNamespace.ArgDescriptor.ArgType>, OpNamespace.ArgDescriptor> standardizeNames
(Map<String, List<ArgDescriptorProposal>> toStandardize, String opName) {
Map<String,List<ArgDescriptorProposal>> ret = new HashMap<>();
Map<Pair<Integer, OpNamespace.ArgDescriptor.ArgType>,List<ArgDescriptorProposal>> dimensionProposals = new HashMap<>();
Map<Pair<Integer, OpNamespace.ArgDescriptor.ArgType>,List<ArgDescriptorProposal>> inPlaceProposals = new HashMap<>();
Map<Pair<Integer, OpNamespace.ArgDescriptor.ArgType>,List<ArgDescriptorProposal>> inputsProposals = new HashMap<>();
Map<Pair<Integer, OpNamespace.ArgDescriptor.ArgType>,List<ArgDescriptorProposal>> inputs2Proposals = new HashMap<>();
Map<Pair<Integer, OpNamespace.ArgDescriptor.ArgType>,List<ArgDescriptorProposal>> outputsProposals = new HashMap<>();
toStandardize.entrySet().forEach(entry -> {
if(entry.getKey().isEmpty()) {
throw new IllegalStateException("Name must not be empty!");
}
if(dimensionNames.contains(entry.getKey())) {
extractProposals(dimensionProposals, entry);
} else if(inplaceNames.contains(entry.getKey())) {
extractProposals(inPlaceProposals, entry);
} else if(inputNames.contains(entry.getKey())) {
extractProposals(inputsProposals, entry);
} else if(input2Names.contains(entry.getKey())) {
extractProposals(inputs2Proposals, entry);
} else if(outputNames.contains(entry.getKey())) {
extractProposals(outputsProposals, entry);
}
else {
ret.put(entry.getKey(),entry.getValue());
}
});
/**
* Two core ops have issues:
* argmax and cumsum both have the same issue
* other boolean attributes are present
* that are converted to ints and get clobbered
* by dimensions having the wrong index
*
* For argmax, exclusive gets clobbered. It should have index 0,
* but for some reason dimensions gets index 0.
*/
/**
* TODO: make this method return name/type
* combinations rather than just name/single list.
*/
if(!dimensionProposals.isEmpty()) {
// List<Pair<Pair<Integer, OpNamespace.ArgDescriptor.ArgType>, ArgDescriptorProposal>> d
computeAggregatedProposalsPerType(ret, dimensionProposals, "dimensions");
}
if(!inPlaceProposals.isEmpty()) {
computeAggregatedProposalsPerType(ret, inPlaceProposals, "inPlace");
}
if(!inputsProposals.isEmpty()) {
computeAggregatedProposalsPerType(ret, inputsProposals, "input");
}
if(!inputs2Proposals.isEmpty()) {
computeAggregatedProposalsPerType(ret, inputs2Proposals, "y");
}
if(!outputsProposals.isEmpty()) {
computeAggregatedProposalsPerType(ret, outputsProposals, "outputs");
}
Map<Pair<Integer, OpNamespace.ArgDescriptor.ArgType>, OpNamespace.ArgDescriptor> ret2 = new HashMap<>();
CounterMap<Pair<Integer, OpNamespace.ArgDescriptor.ArgType>,ArgDescriptorProposal> proposalsByType = new CounterMap<>();
ret.values().forEach(input -> {
input.forEach(proposal1 -> {
proposalsByType.incrementCount(Pair.of(proposal1.getDescriptor().getArgIndex(),proposal1.getDescriptor().getArgType()),proposal1,proposal1.getProposalWeight());
});
});
ret.clear();
proposalsByType.keySet().stream().forEach(argTypeIndexPair -> {
val proposal = proposalsByType.getCounter(argTypeIndexPair).argMax();
val name = proposal.getDescriptor().getName();
List<ArgDescriptorProposal> proposalsForName;
if(!ret.containsKey(name)) {
proposalsForName = new ArrayList<>();
ret.put(name,proposalsForName);
}
else
proposalsForName = ret.get(name);
proposalsForName.add(proposal);
ret.put(proposal.getDescriptor().getName(),proposalsForName);
});
ret.forEach((name,proposals) -> {
val proposalsGroupedByType = proposals.stream().collect(Collectors.groupingBy(proposal -> proposal.getDescriptor().getArgType()));
List<ArgDescriptorProposal> maxProposalsForEachType = new ArrayList<>();
proposalsGroupedByType.forEach((type,proposalGroupByType) -> {
Counter<ArgDescriptorProposal> proposalsCounter = new Counter<>();
proposalGroupByType.forEach(proposalByType -> {
proposalsCounter.incrementCount(proposalByType,proposalByType.getProposalWeight());
});
maxProposalsForEachType.add(proposalsCounter.argMax());
});
proposals = maxProposalsForEachType;
//group by index and type
val collected = proposals.stream()
.collect(Collectors.groupingBy(input -> Pair.of(input.getDescriptor().getArgIndex(),input.getDescriptor().getArgType())))
.entrySet()
.stream().map(input -> Pair.of(input.getKey(),
aggregateProposals(input.getValue()).getDescriptor()))
.collect(Collectors.toMap(pair -> pair.getKey(),pair -> pair.getValue()));
val groupedByType = collected.entrySet().stream().collect(Collectors.groupingBy(input -> input.getKey().getRight()));
groupedByType.forEach((argType,list) -> {
//count number of elements that aren't -1
int numGreaterThanNegativeOne = list.stream().map(input -> input.getKey().getFirst() >= 0 ? 1 : 0)
.reduce(0,(a,b) -> a + b);
if(numGreaterThanNegativeOne > 1) {
throw new IllegalStateException("Name of " + name + " with type " + argType + " not aggregated properly.");
}
});
val arrEntries = collected.entrySet().stream()
.filter(pair -> pair.getValue().getIsArray())
.collect(Collectors.toList());
//process arrays separately and aggregate by type
if(!arrEntries.isEmpty()) {
val initialType = arrEntries.get(0).getValue().getArgType();
val allSameType = new AtomicBoolean(true);
val negativeOnePresent = new AtomicBoolean(false);
arrEntries.forEach(entry -> {
allSameType.set(allSameType.get() && entry.getValue().getArgType() == initialType);
negativeOnePresent.set(negativeOnePresent.get() || entry.getValue().getArgIndex() == -1);
//only remove if we see -1
if(negativeOnePresent.get())
collected.remove(entry.getKey());
});
if(allSameType.get() && negativeOnePresent.get()) {
collected.put(Pair.of(-1,initialType), OpNamespace.ArgDescriptor.newBuilder()
.setArgType(initialType)
.setArgIndex(-1)
.setIsArray(true)
.setName(arrEntries.get(0).getValue().getName()).build());
}
}
ret2.putAll(collected);
});
Map<OpNamespace.ArgDescriptor.ArgType,Integer> maxIndex = new HashMap<>();
if(!bannedMaxIndexOps.contains(opName))
ret2.forEach((key,value) -> {
if(!maxIndex.containsKey(key.getRight())) {
maxIndex.put(key.getValue(),key.getFirst());
} else {
maxIndex.put(key.getValue(),Math.max(key.getFirst(),maxIndex.get(key.getValue())));
}
});
//update -1 values to be valid indices relative to whatever the last index is when an array is found
//and -1 is present
Map<Pair<Integer, OpNamespace.ArgDescriptor.ArgType>, OpNamespace.ArgDescriptor> updateValues = new HashMap<>();
Set<Pair<Integer, OpNamespace.ArgDescriptor.ArgType>> removeKeys = new HashSet<>();
if(!bannedMaxIndexOps.contains(opName))
ret2.forEach((key,value) -> {
if(value.getArgIndex() < 0) {
removeKeys.add(key);
int maxIdx = maxIndex.get(value.getArgType());
updateValues.put(Pair.of(maxIdx + 1,value.getArgType()), OpNamespace.ArgDescriptor.newBuilder()
.setName(value.getName())
.setIsArray(value.getIsArray())
.setArgType(value.getArgType())
.setArgIndex(maxIdx + 1)
.setConvertBoolToInt(value.getConvertBoolToInt())
.build());
}
});
removeKeys.forEach(key -> ret2.remove(key));
ret2.putAll(updateValues);
return ret2;
}
private static void computeAggregatedProposalsPerType(Map<String, List<ArgDescriptorProposal>> ret, Map<Pair<Integer, OpNamespace.ArgDescriptor.ArgType>, List<ArgDescriptorProposal>> dimensionProposals, String name) {
List<ArgDescriptorProposal> dimensions = dimensionProposals.entrySet().stream().map(indexTypeAndList -> {
if(indexTypeAndList.getValue().isEmpty()) {
throw new IllegalStateException("Unable to compute aggregated proposals for an empty list");
}
OpNamespace.ArgDescriptor template = indexTypeAndList.getValue().get(0).getDescriptor();
int idx = indexTypeAndList.getKey().getFirst();
OpNamespace.ArgDescriptor.ArgType type = indexTypeAndList.getKey().getRight();
return Pair.of(indexTypeAndList.getKey(),
ArgDescriptorProposal.builder()
.sourceOfProposal("computedAggregate")
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgIndex(idx)
.setArgType(type)
.setName(name)
.setIsArray(template.getIsArray() || idx < 0)
.build())
.proposalWeight(indexTypeAndList.getValue().stream()
.collect(Collectors.summingDouble(input -> input.getProposalWeight()))
).build());
}).map(input -> input.getSecond()).
collect(Collectors.toList());
ret.put(name,dimensions);
}
private static void extractProposals(Map<Pair<Integer, OpNamespace.ArgDescriptor.ArgType>, List<ArgDescriptorProposal>> inPlaceProposals, Map.Entry<String, List<ArgDescriptorProposal>> entry) {
entry.getValue().forEach(proposal -> {
List<ArgDescriptorProposal> proposals = null;
if (!inPlaceProposals.containsKey(extractKey(proposal))) {
proposals = new ArrayList<>();
inPlaceProposals.put(extractKey(proposal), proposals);
} else {
proposals = inPlaceProposals.get(extractKey(proposal));
}
proposals.add(proposal);
inPlaceProposals.put(extractKey(proposal), proposals);
});
}
/**
* Extract a key reflecting index and type of arg descriptor
* @param proposal the input proposal
* @return
*/
public static Pair<Integer, OpNamespace.ArgDescriptor.ArgType> extractKey(ArgDescriptorProposal proposal) {
return Pair.of(proposal.getDescriptor().getArgIndex(),proposal.getDescriptor().getArgType());
}
public static boolean proposalsAllSameType(List<ArgDescriptorProposal> proposals) {
OpNamespace.ArgDescriptor.ArgType firstType = proposals.get(0).getDescriptor().getArgType();
for(ArgDescriptorProposal proposal : proposals) {
if(proposal.getDescriptor().getArgType() != firstType) {
return false;
}
}
return true;
}
private static List<ArgDescriptorProposal> mergeProposals(Map<String, List<ArgDescriptorProposal>> ret, List<ArgDescriptorProposal> dimensionsList, OpNamespace.ArgDescriptor.ArgType argType, String nameOfArgDescriptor) {
double priorityWeight = 0.0;
ArgDescriptorProposal.ArgDescriptorProposalBuilder newProposalBuilder = ArgDescriptorProposal.builder();
Counter<Integer> indexCounter = new Counter<>();
List<ArgDescriptorProposal> proposalsOutsideType = new ArrayList<>();
boolean allArrayType = true;
for(ArgDescriptorProposal argDescriptorProposal : dimensionsList) {
allArrayType = argDescriptorProposal.getDescriptor().getIsArray() && allArrayType;
//handle arrays separately
if(argDescriptorProposal.getDescriptor().getArgType() == argType) {
indexCounter.incrementCount(argDescriptorProposal.getDescriptor().getArgIndex(),1);
priorityWeight += argDescriptorProposal.getProposalWeight();
} else if(argDescriptorProposal.getDescriptor().getArgType() != argType) {
proposalsOutsideType.add(argDescriptorProposal);
}
}
dimensionsList.clear();
//don't add a list if one is not present
if(!indexCounter.isEmpty()) {
newProposalBuilder
.proposalWeight(priorityWeight)
.descriptor(
OpNamespace.ArgDescriptor.newBuilder()
.setName(nameOfArgDescriptor)
.setArgType(argType)
.setIsArray(allArrayType)
.setArgIndex(indexCounter.argMax())
.build());
dimensionsList.add(newProposalBuilder.build());
ret.put(nameOfArgDescriptor, dimensionsList);
}
//standardize the names
proposalsOutsideType.forEach(proposalOutsideType -> {
proposalOutsideType.setDescriptor(
OpNamespace.ArgDescriptor.newBuilder()
.setName(nameOfArgDescriptor)
.setArgType(proposalOutsideType.getDescriptor().getArgType())
.setArgIndex(proposalOutsideType.getDescriptor().getArgIndex())
.setIsArray(proposalOutsideType.getDescriptor().getIsArray())
.setConvertBoolToInt(proposalOutsideType.getDescriptor().getConvertBoolToInt())
.build()
);
});
return proposalsOutsideType;
}
public static boolean matchesArrayArgDeclaration(String testLine) {
boolean ret = Pattern.matches(ARRAY_ASSIGNMENT,testLine);
return ret;
}
public static boolean matchesArgDeclaration(String argType,String testLine) {
Matcher matcher = Pattern.compile(argType + ARGUMENT_ENDING_PATTERN).matcher(testLine);
Matcher argOnly = Pattern.compile(argType + ARGUMENT_PATTERN).matcher(testLine);
// Matcher arrArg = Pattern.compile(argType + ARGUMENT_PATTERN)
boolean ret = matcher.find();
boolean argOnlyResult = argOnly.find();
return ret || testLine.contains("?") && argOnlyResult
|| testLine.contains("static_cast") && argOnlyResult
|| (testLine.contains("))") && argOnlyResult && !testLine.contains("if") && !testLine.contains("REQUIRE_TRUE")) && !testLine.contains("->rankOf()")
|| (testLine.contains("==") && argOnlyResult && !testLine.contains("if") && !testLine.contains("REQUIRE_TRUE")) && !testLine.contains("->rankOf()")
|| (testLine.contains("(" + argType) && argOnlyResult && !testLine.contains("if") && !testLine.contains("REQUIRE_TRUE")) && !testLine.contains("->rankOf()")
|| (testLine.contains("->") && argOnlyResult && !testLine.contains("if") && !testLine.contains("REQUIRE_TRUE")) && !testLine.contains("->rankOf()");
}
}
@@ -0,0 +1,999 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.nd4j.descriptor.proposal.impl;
import com.github.javaparser.ParserConfiguration;
import com.github.javaparser.StaticJavaParser;
import com.github.javaparser.ast.CompilationUnit;
import com.github.javaparser.ast.body.ConstructorDeclaration;
import com.github.javaparser.ast.body.FieldDeclaration;
import com.github.javaparser.ast.expr.MethodCallExpr;
import com.github.javaparser.resolution.declarations.ResolvedConstructorDeclaration;
import com.github.javaparser.resolution.declarations.ResolvedFieldDeclaration;
import com.github.javaparser.resolution.declarations.ResolvedParameterDeclaration;
import com.github.javaparser.symbolsolver.JavaSymbolSolver;
import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver;
import com.github.javaparser.symbolsolver.resolution.typesolvers.JavaParserTypeSolver;
import com.github.javaparser.symbolsolver.resolution.typesolvers.ReflectionTypeSolver;
import com.github.javaparser.utils.Log;
import com.github.javaparser.utils.SourceRoot;
import lombok.Builder;
import lombok.Getter;
import org.nd4j.autodiff.functions.DifferentialFunction;
import org.nd4j.autodiff.samediff.SDVariable;
import org.nd4j.common.primitives.Counter;
import org.nd4j.common.primitives.CounterMap;
import org.nd4j.common.primitives.Pair;
import org.nd4j.descriptor.proposal.ArgDescriptorProposal;
import org.nd4j.descriptor.proposal.ArgDescriptorSource;
import org.nd4j.ir.OpNamespace;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.*;
import org.nd4j.linalg.api.ops.impl.indexaccum.custom.ArgMax;
import org.nd4j.linalg.api.ops.impl.indexaccum.custom.ArgMin;
import org.nd4j.linalg.api.ops.impl.transforms.BaseDynamicTransformOp;
import org.reflections.Reflections;
import java.io.File;
import java.lang.reflect.Modifier;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import static org.nd4j.descriptor.proposal.impl.ArgDescriptorParserUtils.*;
public class JavaSourceArgDescriptorSource implements ArgDescriptorSource {
private SourceRoot sourceRoot;
private File nd4jOpsRootDir;
private double weight;
/**
* void addTArgument(double... arg);
*
* void addIArgument(int... arg);
*
* void addIArgument(long... arg);
*
* void addBArgument(boolean... arg);
*
* void addDArgument(DataType... arg);
*/
public final static String ADD_T_ARGUMENT_INVOCATION = "addTArgument";
public final static String ADD_I_ARGUMENT_INVOCATION = "addIArgument";
public final static String ADD_B_ARGUMENT_INVOCATION = "addBArgument";
public final static String ADD_D_ARGUMENT_INVOCATION = "addDArgument";
public final static String ADD_INPUT_ARGUMENT = "addInputArgument";
public final static String ADD_OUTPUT_ARGUMENT = "addOutputArgument";
@Getter
private Map<String, OpNamespace.OpDescriptor.OpDeclarationType> opTypes;
static {
Log.setAdapter(new Log.StandardOutStandardErrorAdapter());
}
@Builder
public JavaSourceArgDescriptorSource(File nd4jApiRootDir,double weight) {
this.sourceRoot = initSourceRoot(nd4jApiRootDir);
this.nd4jOpsRootDir = nd4jApiRootDir;
if(opTypes == null) {
opTypes = new HashMap<>();
}
this.weight = weight;
}
public Map<String, List<ArgDescriptorProposal>> doReflectionsExtraction() {
Map<String, List<ArgDescriptorProposal>> ret = new HashMap<>();
Reflections reflections = new Reflections("org.nd4j");
Set<Class<? extends DifferentialFunction>> subTypesOf = reflections.getSubTypesOf(DifferentialFunction.class);
Set<Class<? extends CustomOp>> subTypesOfOp = reflections.getSubTypesOf(CustomOp.class);
Set<Class<?>> allClasses = new HashSet<>();
allClasses.addAll(subTypesOf);
allClasses.addAll(subTypesOfOp);
Set<String> opNamesForDifferentialFunction = new HashSet<>();
for(Class<?> clazz : allClasses) {
if(Modifier.isAbstract(clazz.getModifiers()) || clazz.isInterface()) {
continue;
}
processClazz(ret, opNamesForDifferentialFunction, clazz);
}
return ret;
}
private void processClazz(Map<String, List<ArgDescriptorProposal>> ret, Set<String> opNamesForDifferentialFunction, Class<?> clazz) {
try {
Object funcInstance = clazz.newInstance();
String name = null;
if(funcInstance instanceof DifferentialFunction) {
DifferentialFunction differentialFunction = (DifferentialFunction) funcInstance;
name = differentialFunction.opName();
} else if(funcInstance instanceof CustomOp) {
CustomOp customOp = (CustomOp) funcInstance;
name = customOp.opName();
}
if(name == null)
return;
opNamesForDifferentialFunction.add(name);
if(!(funcInstance instanceof DynamicCustomOp))
opTypes.put(name,OpNamespace.OpDescriptor.OpDeclarationType.LEGACY_XYZ);
else
opTypes.put(name,OpNamespace.OpDescriptor.OpDeclarationType.CUSTOM_OP_IMPL);
String fileName = clazz.getName().replace(".",File.separator);
StringBuilder fileBuilder = new StringBuilder();
fileBuilder.append(fileName);
fileBuilder.append(".java");
CounterMap<Pair<String, OpNamespace.ArgDescriptor.ArgType>,Integer> paramIndicesCount = new CounterMap<>();
// Our sample is in the root of this directory, so no package name.
CompilationUnit cu = sourceRoot.parse(clazz.getPackage().getName(), clazz.getSimpleName() + ".java");
cu.findAll(MethodCallExpr.class).forEach(method -> {
String methodInvoked = method.getNameAsString();
final AtomicInteger indexed = new AtomicInteger(0);
//need to figure out how to consolidate multiple method calls
//as well as the right indices
//typical patterns in the code base will reflect adding arguments all at once
//one thing we can just check for is if more than 1 argument is passed in and
//treat that as a complete list of arguments
if(methodInvoked.equals(ADD_T_ARGUMENT_INVOCATION)) {
method.getArguments().forEach(argument -> {
if(argument.isNameExpr())
paramIndicesCount.incrementCount(Pair.of(argument.asNameExpr().getNameAsString(), OpNamespace.ArgDescriptor.ArgType.DOUBLE),indexed.get(),100.0);
else if(argument.isMethodCallExpr()) {
if(argument.asMethodCallExpr().getName().toString().equals("ordinal")) {
paramIndicesCount.incrementCount(Pair.of(argument.asNameExpr().getNameAsString(), OpNamespace.ArgDescriptor.ArgType.DOUBLE),indexed.get(),100.0);
}
}
indexed.incrementAndGet();
});
} else if(methodInvoked.equals(ADD_B_ARGUMENT_INVOCATION)) {
method.getArguments().forEach(argument -> {
if(argument.isNameExpr())
paramIndicesCount.incrementCount(Pair.of(argument.asNameExpr().getNameAsString(), OpNamespace.ArgDescriptor.ArgType.BOOL),indexed.get(),100.0);
else if(argument.isMethodCallExpr()) {
if(argument.asMethodCallExpr().getName().equals("ordinal")) {
paramIndicesCount.incrementCount(Pair.of(argument.asNameExpr().getNameAsString(), OpNamespace.ArgDescriptor.ArgType.BOOL),indexed.get(),100.0);
}
}
indexed.incrementAndGet();
});
} else if(methodInvoked.equals(ADD_I_ARGUMENT_INVOCATION)) {
method.getArguments().forEach(argument -> {
if(argument.isNameExpr())
paramIndicesCount.incrementCount(Pair.of(argument.asNameExpr().getNameAsString(), OpNamespace.ArgDescriptor.ArgType.INT64),indexed.get(),100.0);
else if(argument.isMethodCallExpr()) {
if(argument.asMethodCallExpr().getName().toString().equals("ordinal")) {
paramIndicesCount.incrementCount(Pair.of(argument.toString().replace(".ordinal()",""), OpNamespace.ArgDescriptor.ArgType.INT64),indexed.get(),100.0);
}
}
indexed.incrementAndGet();
});
} else if(methodInvoked.equals(ADD_D_ARGUMENT_INVOCATION)) {
method.getArguments().forEach(argument -> {
if(argument.isNameExpr())
paramIndicesCount.incrementCount(Pair.of(argument.asNameExpr().getNameAsString(), OpNamespace.ArgDescriptor.ArgType.DATA_TYPE),indexed.get(),100.0);
else if(argument.isMethodCallExpr()) {
if(argument.asMethodCallExpr().getName().toString().equals("ordinal")) {
paramIndicesCount.incrementCount(Pair.of(argument.toString().replace(".ordinal()",""), OpNamespace.ArgDescriptor.ArgType.DATA_TYPE),indexed.get(),100.0);
}
}
indexed.incrementAndGet();
});
} else if(methodInvoked.equals(ADD_INPUT_ARGUMENT)) {
method.getArguments().forEach(argument -> {
if(argument.isNameExpr())
paramIndicesCount.incrementCount(Pair.of(argument.asNameExpr().getNameAsString(), OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR),indexed.get(),100.0);
else if(argument.isMethodCallExpr()) {
if(argument.asMethodCallExpr().getName().toString().equals("ordinal")) {
paramIndicesCount.incrementCount(Pair.of(argument.toString().replace(".ordinal()",""), OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR),indexed.get(),100.0);
}
}
indexed.incrementAndGet();
});
} else if(methodInvoked.equals(ADD_OUTPUT_ARGUMENT)) {
method.getArguments().forEach(argument -> {
if(argument.isNameExpr())
paramIndicesCount.incrementCount(Pair.of(argument.asNameExpr().getNameAsString(), OpNamespace.ArgDescriptor.ArgType.OUTPUT_TENSOR),indexed.get(),100.0);
else if(argument.isMethodCallExpr()) {
if(argument.asMethodCallExpr().getName().toString().equals("ordinal")) {
paramIndicesCount.incrementCount(Pair.of(argument.toString().replace(".ordinal()",""), OpNamespace.ArgDescriptor.ArgType.OUTPUT_TENSOR),indexed.get(),100.0);
}
}
indexed.incrementAndGet();
});
}
}
);
List<ResolvedConstructorDeclaration> collect = cu.findAll(ConstructorDeclaration.class).stream()
.map(input -> input.resolve())
.filter(constructor -> constructor.getNumberOfParams() > 0)
.distinct()
.collect(Collectors.toList());
//only process final constructor with all arguments for indexing purposes
Counter<ResolvedConstructorDeclaration> constructorArgCount = new Counter<>();
collect.stream().filter(input -> input != null).forEach(constructor -> {
constructorArgCount.incrementCount(constructor,constructor.getNumberOfParams());
});
if(constructorArgCount.argMax() != null)
collect = Arrays.asList(constructorArgCount.argMax());
List<ArgDescriptorProposal> argDescriptorProposals = ret.get(name);
if(argDescriptorProposals == null) {
argDescriptorProposals = new ArrayList<>();
ret.put(name,argDescriptorProposals);
}
Set<ResolvedParameterDeclaration> parameters = new LinkedHashSet<>();
int floatIdx = 0;
int inputIdx = 0;
int outputIdx = 0;
int intIdx = 0;
int boolIdx = 0;
int dTypeIndex = 0;
for(ResolvedConstructorDeclaration parameterDeclaration : collect) {
floatIdx = 0;
inputIdx = 0;
outputIdx = 0;
intIdx = 0;
boolIdx = 0;
dTypeIndex = 0;
for(int i = 0; i < parameterDeclaration.getNumberOfParams(); i++) {
ResolvedParameterDeclaration param = parameterDeclaration.getParam(i);
OpNamespace.ArgDescriptor.ArgType argType = argTypeForParam(param);
if(isValidParam(param)) {
parameters.add(param);
switch(argType) {
case INPUT_TENSOR:
paramIndicesCount.incrementCount(Pair.of(param.getName(),argType), inputIdx, 100.0);
inputIdx++;
break;
case INT64:
case INT32:
paramIndicesCount.incrementCount(Pair.of(param.getName(), OpNamespace.ArgDescriptor.ArgType.INT64), intIdx, 100.0);
intIdx++;
break;
case DATA_TYPE:
paramIndicesCount.incrementCount(Pair.of(param.getName(), OpNamespace.ArgDescriptor.ArgType.DATA_TYPE), dTypeIndex, 100.0);
dTypeIndex++;
break;
case DOUBLE:
case FLOAT:
paramIndicesCount.incrementCount(Pair.of(param.getName(), OpNamespace.ArgDescriptor.ArgType.FLOAT), floatIdx, 100.0);
paramIndicesCount.incrementCount(Pair.of(param.getName(), OpNamespace.ArgDescriptor.ArgType.DOUBLE), floatIdx, 100.0);
floatIdx++;
break;
case BOOL:
paramIndicesCount.incrementCount(Pair.of(param.getName(),argType), boolIdx, 100.0);
boolIdx++;
break;
case OUTPUT_TENSOR:
paramIndicesCount.incrementCount(Pair.of(param.getName(),argType), outputIdx, 100.0);
outputIdx++;
break;
case UNRECOGNIZED:
continue;
}
}
}
}
floatIdx = 0;
inputIdx = 0;
outputIdx = 0;
intIdx = 0;
boolIdx = 0;
Set<List<Pair<String, String>>> typesAndParams = parameters.stream().map(collectedParam ->
Pair.of(collectedParam.describeType(), collectedParam.getName()))
.collect(Collectors.groupingBy(input -> input.getSecond())).entrySet()
.stream()
.map(inputPair -> inputPair.getValue())
.collect(Collectors.toSet());
Set<String> constructorNamesEncountered = new HashSet<>();
List<ArgDescriptorProposal> finalArgDescriptorProposals = argDescriptorProposals;
typesAndParams.forEach(listOfTypesAndNames -> {
listOfTypesAndNames.forEach(parameter -> {
if(typeNameOrArrayOfTypeNameMatches(parameter.getFirst(),SDVariable.class.getName(),INDArray.class.getName())) {
constructorNamesEncountered.add(parameter.getValue());
if(outputNames.contains(parameter.getValue())) {
Counter<Integer> counter = paramIndicesCount.getCounter(Pair.of(parameter.getSecond(), OpNamespace.ArgDescriptor.ArgType.OUTPUT_TENSOR));
if(counter != null)
finalArgDescriptorProposals.add(ArgDescriptorProposal.builder()
.proposalWeight(99.0 * (counter == null ? 1 : counter.size()))
.sourceOfProposal("java")
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.OUTPUT_TENSOR)
.setName(parameter.getSecond())
.setIsArray(parameter.getFirst().contains("[]") || parameter.getFirst().contains("..."))
.setArgIndex(counter.argMax())
.build()).build());
} else {
Counter<Integer> counter = paramIndicesCount.getCounter(Pair.of(parameter.getSecond(), OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR));
if(counter != null)
finalArgDescriptorProposals.add(ArgDescriptorProposal.builder()
.proposalWeight(99.0 * (counter == null ? 1 : counter.size()))
.sourceOfProposal("java")
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName(parameter.getSecond())
.setIsArray(parameter.getFirst().contains("[]") || parameter.getFirst().contains("..."))
.setArgIndex(counter.argMax())
.build()).build());
}
} else if(typeNameOrArrayOfTypeNameMatches(parameter.getFirst(),int.class.getName(),long.class.getName(),Integer.class.getName(),Long.class.getName()) || paramIsEnum(parameter.getFirst())) {
constructorNamesEncountered.add(parameter.getValue());
Counter<Integer> counter = paramIndicesCount.getCounter(Pair.of(parameter.getSecond(), OpNamespace.ArgDescriptor.ArgType.INT64));
if(counter != null)
finalArgDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(99.0 * (counter == null ? 1 : counter.size()))
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName(parameter.getSecond())
.setIsArray(parameter.getFirst().contains("[]") || parameter.getFirst().contains("..."))
.setArgIndex(counter.argMax())
.build()).build());
} else if(typeNameOrArrayOfTypeNameMatches(parameter.getFirst(),float.class.getName(),double.class.getName(),Float.class.getName(),Double.class.getName())) {
constructorNamesEncountered.add(parameter.getValue());
Counter<Integer> counter = paramIndicesCount.getCounter(Pair.of(parameter.getSecond(), OpNamespace.ArgDescriptor.ArgType.FLOAT));
if(counter != null)
finalArgDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(99.0 * (counter == null ? 1 :(counter == null ? 1 : counter.size()) ))
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.DOUBLE)
.setName(parameter.getSecond())
.setIsArray(parameter.getFirst().contains("[]"))
.setArgIndex(counter.argMax())
.build()).build());
} else if(typeNameOrArrayOfTypeNameMatches(parameter.getFirst(),boolean.class.getName(),Boolean.class.getName())) {
constructorNamesEncountered.add(parameter.getValue());
Counter<Integer> counter = paramIndicesCount.getCounter(Pair.of(parameter.getSecond(), OpNamespace.ArgDescriptor.ArgType.BOOL));
if(counter != null)
finalArgDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(99.0 * (counter == null ? 1 :(counter == null ? 1 : counter.size()) ))
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.BOOL)
.setName(parameter.getSecond())
.setIsArray(parameter.getFirst().contains("[]"))
.setArgIndex(counter.argMax())
.build()).build());
}
});
});
List<ResolvedFieldDeclaration> fields = cu.findAll(FieldDeclaration.class).stream()
.map(input -> getResolve(input))
//filter fields
.filter(input -> input != null && !input.isStatic())
.collect(Collectors.toList());
floatIdx = 0;
inputIdx = 0;
outputIdx = 0;
intIdx = 0;
boolIdx = 0;
for(ResolvedFieldDeclaration field : fields) {
if(!constructorNamesEncountered.contains(field.getName()) && typeNameOrArrayOfTypeNameMatches(field.getType().describe(),SDVariable.class.getName(),INDArray.class.getName())) {
if(outputNames.contains(field.getName())) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(99.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.OUTPUT_TENSOR)
.setName(field.getName())
.setIsArray(field.getType().describe().contains("[]"))
.setArgIndex(outputIdx)
.build()).build());
outputIdx++;
} else if(!constructorNamesEncountered.contains(field.getName())){
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(99.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName(field.getName())
.setIsArray(field.getType().describe().contains("[]"))
.setArgIndex(inputIdx)
.build()).build());
inputIdx++;
}
} else if(!constructorNamesEncountered.contains(field.getName()) && typeNameOrArrayOfTypeNameMatches(field.getType().describe(),int.class.getName(),long.class.getName(),Long.class.getName(),Integer.class.getName())) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(99.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName(field.getName())
.setIsArray(field.getType().describe().contains("[]"))
.setArgIndex(intIdx)
.build()).build());
intIdx++;
} else if(!constructorNamesEncountered.contains(field.getName()) && typeNameOrArrayOfTypeNameMatches(field.getType().describe(),double.class.getName(),float.class.getName(),Double.class.getName(),Float.class.getName())) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(99.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.DOUBLE)
.setName(field.getName())
.setIsArray(field.getType().describe().contains("[]"))
.setArgIndex(floatIdx)
.build()).build());
floatIdx++;
} else if(!constructorNamesEncountered.contains(field.getName()) && typeNameOrArrayOfTypeNameMatches(field.getType().describe(),Boolean.class.getName(),boolean.class.getName())) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(99.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.BOOL)
.setName(field.getName())
.setIsArray(field.getType().describe().contains("[]"))
.setArgIndex(boolIdx)
.build()).build());
boolIdx++;
}
}
if(funcInstance instanceof BaseReduceOp ||
funcInstance instanceof BaseReduceBoolOp || funcInstance instanceof BaseReduceSameOp) {
if(!containsProposalWithDescriptorName("keepDims",argDescriptorProposals)) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(9999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.BOOL)
.setName("keepDims")
.setIsArray(false)
.setArgIndex(boolIdx)
.build()).build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(9999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName("dimensions")
.setIsArray(true)
.setArgIndex(intIdx)
.build()).build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(9999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName("dimensions")
.setIsArray(false)
.setArgIndex(1)
.build()).build());
}
if(funcInstance instanceof ArgMax || funcInstance instanceof ArgMin) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(99999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName("dimensions")
.setIsArray(true)
.setArgIndex(intIdx)
.build()).build());
}
if(!containsProposalWithDescriptorName("dimensions",argDescriptorProposals)) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(9999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName("dimensions")
.setIsArray(true)
.setArgIndex(0)
.build()).build());
}
}
if(funcInstance instanceof BaseTransformBoolOp) {
BaseTransformBoolOp baseTransformBoolOp = (BaseTransformBoolOp) funcInstance;
if(baseTransformBoolOp.getOpType() == Op.Type.PAIRWISE_BOOL) {
if(numProposalsWithType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR,argDescriptorProposals) < 2) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(9999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName("y")
.setIsArray(false)
.setArgIndex(1)
.build()).build());
}
}
}
if(funcInstance instanceof BaseDynamicTransformOp) {
if(!containsProposalWithDescriptorName("inPlace",argDescriptorProposals)) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(9999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.BOOL)
.setName("inPlace")
.setIsArray(false)
.setArgIndex(boolIdx)
.build()).build());
}
}
//hard coded case, impossible to parse from as the code exists today, and it doesn't exist anywhere in the libnd4j code base
if(name.contains("maxpool2d")) {
if(!containsProposalWithDescriptorName("extraParam0",argDescriptorProposals)) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("extraParam0")
.proposalWeight(9999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName("extraParam0")
.setIsArray(false)
.setArgIndex(9)
.build()).build());
}
}
if(name.contains("scatter_update")) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName("indices")
.setIsArray(false)
.setArgIndex(2)
.build()).build());
}
if(name.contains("fill")) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName("shape")
.setIsArray(false)
.setArgIndex(0)
.build()).build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName("result")
.setIsArray(false)
.setArgIndex(1)
.build()).build());
}
if(name.contains("loop_cond")) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(9999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName("input")
.setIsArray(false)
.setArgIndex(0)
.build()).build());
}
if(name.equals("top_k")) {
if(!containsProposalWithDescriptorName("sorted",argDescriptorProposals)) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("sorted")
.proposalWeight(9999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName("sorted")
.setIsArray(false)
.setArgIndex(0)
.build()).build());
}
}
//dummy output tensor
if(name.equals("next_iteration")) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.proposalWeight(9999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder().setArgIndex(0)
.setArgType(OpNamespace.ArgDescriptor.ArgType.OUTPUT_TENSOR)
.setName("output").build())
.build());
}
if(!containsOutputTensor(argDescriptorProposals)) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("z")
.proposalWeight(9999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.OUTPUT_TENSOR)
.setName("z")
.setIsArray(false)
.setArgIndex(0)
.build()).build());
}
if(name.equals("gather")) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("axis")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName("axis")
.setIsArray(false)
.setArgIndex(0)
.build()).build());
}
if(name.equals("pow")) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("pow")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName("pow")
.setIsArray(false)
.setArgIndex(1)
.build()).build());
}
if(name.equals("concat")) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("isDynamicAxis")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.BOOL)
.setName("isDynamicAxis")
.setIsArray(false)
.setArgIndex(0)
.build()).build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("concatDimension")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName("isDynamicAxis")
.setIsArray(false)
.setArgIndex(1)
.build()).build());
}
if(name.equals("merge")) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.proposalWeight(99999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder().setArgIndex(0)
.setIsArray(true)
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName("inputs").build())
.build());
}
if(name.equals("split") || name.equals("split_v")) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("numSplit")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName("numSplit")
.setIsArray(false)
.setArgIndex(0)
.build()).build());
}
if(name.equals("reshape")) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("shape")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName("shape")
.setIsArray(false)
.setArgIndex(0)
.build()).build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("shape")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName("shape")
.setIsArray(false)
.setArgIndex(1)
.build()).build());
}
if(name.equals("create")) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.DATA_TYPE)
.setName("outputType")
.setIsArray(false)
.setArgIndex(0)
.build()).build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName("order")
.setIsArray(false)
.setArgIndex(0)
.build()).build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("java")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName("outputType")
.setIsArray(false)
.setArgIndex(1)
.build()).build());
}
if(name.equals("eye")) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("numRows")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName("numRows")
.setIsArray(false)
.setArgIndex(0)
.build()).build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("numCols")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName("numCols")
.setIsArray(false)
.setArgIndex(1)
.build()).build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("batchDimension")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName("batchDimension")
.setIsArray(true)
.setArgIndex(2)
.build()).build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("dataType")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INT64)
.setName("dataType")
.setIsArray(false)
.setArgIndex(3)
.build()).build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("dataType")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.DOUBLE)
.setName("dataType")
.setIsArray(true)
.setArgIndex(0)
.build()).build());
}
if(name.equals("bincount")) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("cpp")
.proposalWeight(Double.MAX_VALUE)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.DATA_TYPE)
.setName("outputType")
.setIsArray(false)
.setArgIndex(0)
.build()).build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("cpp")
.proposalWeight(Double.POSITIVE_INFINITY)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName("values")
.setIsArray(false)
.setArgIndex(0)
.build()).build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("cpp")
.proposalWeight(Double.POSITIVE_INFINITY)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName("weights")
.setIsArray(false)
.setArgIndex(1)
.build()).build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("cpp")
.proposalWeight(Double.POSITIVE_INFINITY)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName("min")
.setIsArray(false)
.setArgIndex(2)
.build()).build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.sourceOfProposal("cpp")
.proposalWeight(Double.POSITIVE_INFINITY)
.descriptor(OpNamespace.ArgDescriptor.newBuilder()
.setArgType(OpNamespace.ArgDescriptor.ArgType.INPUT_TENSOR)
.setName("max")
.setIsArray(false)
.setArgIndex(3)
.build()).build());
}
if(name.equals("while") || name.equals("enter") || name.equals("exit") || name.equals("next_iteration")
|| name.equals("loop_cond") || name.equals("switch") || name.equals("While")) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.proposalWeight(9999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder().setArgIndex(0)
.setArgType(OpNamespace.ArgDescriptor.ArgType.STRING)
.setName("frameName").build())
.build());
}
if(name.equals("resize_bilinear")) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.proposalWeight(99999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder().setArgIndex(0)
.setArgType(OpNamespace.ArgDescriptor.ArgType.BOOL)
.setName("alignCorners").build())
.build());
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.proposalWeight(99999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder().setArgIndex(1)
.setArgType(OpNamespace.ArgDescriptor.ArgType.BOOL)
.setName("halfPixelCenters").build())
.build());
}
if(funcInstance instanceof BaseTransformSameOp || funcInstance instanceof BaseTransformOp || funcInstance instanceof BaseDynamicTransformOp) {
argDescriptorProposals.add(ArgDescriptorProposal.builder()
.proposalWeight(9999.0)
.descriptor(OpNamespace.ArgDescriptor.newBuilder().setArgIndex(0)
.setArgType(OpNamespace.ArgDescriptor.ArgType.DATA_TYPE)
.setName("dataType").build())
.build());
}
} catch(Exception e) {
e.printStackTrace();
}
}
private static ResolvedFieldDeclaration getResolve(FieldDeclaration input) {
try {
return input.resolve();
}catch(Exception e) {
return null;
}
}
private SourceRoot initSourceRoot(File nd4jApiRootDir) {
CombinedTypeSolver typeSolver = new CombinedTypeSolver();
typeSolver.add(new ReflectionTypeSolver(false));
typeSolver.add(new JavaParserTypeSolver(nd4jApiRootDir));
JavaSymbolSolver symbolSolver = new JavaSymbolSolver(typeSolver);
StaticJavaParser.getConfiguration().setSymbolResolver(symbolSolver);
SourceRoot sourceRoot = new SourceRoot(nd4jApiRootDir.toPath(),new ParserConfiguration().setSymbolResolver(symbolSolver));
return sourceRoot;
}
@Override
public Map<String, List<ArgDescriptorProposal>> getProposals() {
return doReflectionsExtraction();
}
@Override
public OpNamespace.OpDescriptor.OpDeclarationType typeFor(String name) {
return opTypes.get(name);
}
}