chore: import upstream snapshot with attribution

codegen/op-codegen/src/main/java/org/nd4j/codegen/Namespace.java

@@ -0,0 +1,146 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen;
+
+import org.nd4j.codegen.api.NamespaceOps;
+import org.nd4j.codegen.ops.*;
+
+public enum Namespace {
+    BITWISE,
+    NEURALNETWORK,
+    RANDOM,
+    IMAGE,
+    CNN,
+    RNN,
+    MATH,
+    BASE,
+    LOSS,
+    LINALG;
+
+
+    public static Namespace fromString(String in){
+        switch (in.toLowerCase()){
+            case "bitwise":
+                return BITWISE;
+            case "nn":
+            case "neuralnetwork":
+                return NEURALNETWORK;
+            case "random":
+                return RANDOM;
+            case "math":
+                return MATH;
+            case "image":
+                return IMAGE;
+            case "cnn":
+                return CNN;
+            case "rnn":
+                return RNN;
+            case "base":
+                return BASE;
+            case "loss":
+                return LOSS;
+            case "linalg":
+                return LINALG;
+            default:
+                return null;
+        }
+    }
+
+    public String javaClassName() {
+        switch (this){
+            case BITWISE:
+                return "NDBitwise";
+            case NEURALNETWORK:
+                return "NDNN";
+            case RANDOM:
+                return "NDRandom";
+            case IMAGE:
+                return "NDImage";
+            case CNN:
+                return "NDCNN";
+            case RNN:
+                return "NDRNN";
+            case MATH:
+                return "NDMath";
+            case BASE:
+                return "NDBase";
+            case LOSS:
+                return "NDLoss";
+            case LINALG:
+                return "NDLinalg";
+        }
+        throw new IllegalStateException("No java class name defined for: " + this);
+    }
+
+    public String javaSameDiffClassName() {
+        switch (this){
+            case BITWISE:
+                return "SDBitwise";
+            case NEURALNETWORK:
+                return "SDNN";
+            case RANDOM:
+                return "SDRandom";
+            case IMAGE:
+                return "SDImage";
+            case CNN:
+                return "SDCNN";
+            case RNN:
+                return "SDRNN";
+            case MATH:
+                return "SDMath";
+            case BASE:
+                return "SDBaseOps";
+            case LOSS:
+                return "SDLoss";
+            /*case VALIDATION:
+                return "SDValidation";*/
+            case LINALG:
+                return "SDLinalg";
+        }
+        throw new IllegalStateException("No java SameDiff class name defined for: " + this);
+    }
+
+    public NamespaceOps getNamespace() {
+        switch (this){
+            case BITWISE:
+                return BitwiseKt.Bitwise();
+            case RANDOM:
+                return RandomKt.Random();
+            case MATH:
+                return MathKt.Math();
+            case IMAGE:
+                return ImageKt.SDImage();
+            case CNN:
+                return CNNKt.SDCNN();
+            case RNN:
+                return RNNKt.SDRNN();
+            case NEURALNETWORK:
+                return NeuralNetworkKt.NN();
+            case BASE:
+                return SDBaseOpsKt.SDBaseOps();
+            case LOSS:
+                return SDLossKt.SDLoss();
+            case LINALG:
+                return LinalgKt.Linalg();
+        }
+        throw new IllegalStateException("No namespace definition available for: " + this);
+    }
+}

codegen/op-codegen/src/main/java/org/nd4j/codegen/cli/CLI.java

@@ -0,0 +1,185 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.cli;
+
+import com.beust.jcommander.IParameterValidator;
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.nd4j.codegen.Namespace;
+import org.nd4j.codegen.api.NamespaceOps;
+import org.nd4j.codegen.impl.java.DocsGenerator;
+import org.nd4j.codegen.impl.java.Nd4jNamespaceGenerator;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Planned CLI for generating classes
+ */
+@Slf4j
+public class CLI {
+    private static final String relativePath = "nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/";
+    private static final String allProjects = "all";
+    private static final String sdProject = "sd";
+    private static final String ndProject = "nd4j";
+
+    public static class ProjectsValidator implements IParameterValidator {
+
+        @Override
+        public void validate(String name, String value) throws ParameterException {
+            if (name.equals("-projects")) {
+                if (!(value.equals(allProjects) || value.equals(ndProject) || value.equals(sdProject))) {
+                    throw new ParameterException("Wrong projects " + value + "  passed! Must be one of [all, sd, nd4j]");
+                }
+            }
+        }
+    }
+
+    @Parameter(names = "-dir", description = "Root directory of deeplearning4j mono repo")
+    private String repoRootDir;
+
+    @Parameter(names = "-docsdir", description = "Root directory for generated docs")
+    private String docsdir;
+
+    @Parameter(names = "-namespaces", description = "List of namespaces to generate, or 'ALL' to generate all namespaces", required = true)
+    private List<String> namespaces;
+
+    @Parameter(names = "-projects", description = "List of sub-projects - ND4J, SameDiff or both", required = false, validateWith = ProjectsValidator.class)
+    private List<String> projects = Collections.singletonList("all");
+
+    enum NS_PROJECT {
+        ND4J,
+        SAMEDIFF;
+    }
+
+    private void generateNamespaces(NS_PROJECT project, File outputDir, String basePackage) throws IOException {
+
+        List<Namespace> usedNamespaces = new ArrayList<>();
+
+        for(String s : namespaces) {
+            if ("all".equalsIgnoreCase(s)) {
+                Collections.addAll(usedNamespaces, Namespace.values());
+                break;
+            }
+            Namespace ns = null;
+            if (project == NS_PROJECT.ND4J) {
+                ns = Namespace.fromString(s);
+                if (ns == null) {
+                    log.error("Invalid/unknown ND4J namespace provided: " + s);
+                }
+                else {
+                    usedNamespaces.add(ns);
+                }
+            }
+            else {
+                ns = Namespace.fromString(s);
+                if (ns == null) {
+                    log.error("Invalid/unknown SD namespace provided: " + s);
+                }
+                else {
+                    usedNamespaces.add(ns);
+                }
+            }
+        }
+
+        int cnt = 0;
+        for (int i = 0; i < usedNamespaces.size(); ++i) {
+            Namespace ns = usedNamespaces.get(i);
+            log.info("Starting generation of namespace: {}", ns);
+
+            String javaClassName = project == NS_PROJECT.ND4J ? ns.javaClassName() : ns.javaSameDiffClassName();
+            NamespaceOps ops = ns.getNamespace();
+
+            String basePackagePath = basePackage.replace(".", "/") + "/ops/";
+
+            if (StringUtils.isNotEmpty(docsdir)) {
+                DocsGenerator.generateDocs(i, ops, docsdir, basePackage);
+            }
+            if (outputDir != null) {
+                File outputPath = new File(outputDir, basePackagePath + javaClassName + ".java");
+                log.info("Output path: {}", outputPath.getAbsolutePath());
+
+                if (NS_PROJECT.ND4J == project)
+                    Nd4jNamespaceGenerator.generate(ops, null, outputDir, javaClassName, basePackage, docsdir);
+                else
+                    Nd4jNamespaceGenerator.generate(ops, null, outputDir, javaClassName, basePackage,
+                            "org.nd4j.autodiff.samediff.ops.SDOps", docsdir);
+            }
+            ++cnt;
+        }
+        log.info("Complete - generated {} namespaces", cnt);
+    }
+
+
+    public static void main(String[] args) throws Exception {
+        new CLI().runMain(args);
+    }
+
+    public void runMain(String[] args) throws Exception {
+        JCommander.newBuilder()
+                .addObject(this)
+                .build()
+                .parse(args);
+
+        // Either root directory for source code generation or docs directory must be present. If root directory is
+        // absenbt - then it's "generate docs only" mode.
+        if (StringUtils.isEmpty(repoRootDir) && StringUtils.isEmpty(docsdir)) {
+            throw new IllegalStateException("Provide one or both of arguments : -dir, -docsdir");
+        }
+        File outputDir = null;
+        if (StringUtils.isNotEmpty(repoRootDir)) {
+            //First: Check root directory.
+            File dir = new File(repoRootDir);
+            if (!dir.exists() || !dir.isDirectory()) {
+                throw new IllegalStateException("Provided root directory does not exist (or not a directory): " + dir.getAbsolutePath());
+            }
+
+            outputDir = new File(dir, relativePath);
+            if (!outputDir.exists() || !dir.isDirectory()) {
+                throw new IllegalStateException("Expected output directory does not exist: " + outputDir.getAbsolutePath());
+            }
+        }
+
+        if(namespaces == null || namespaces.isEmpty()){
+            throw new IllegalStateException("No namespaces were provided");
+        }
+
+        try {
+            if (projects == null)
+                projects.add(allProjects);
+            boolean forAllProjects = projects.isEmpty() || projects.contains(allProjects);
+            if (forAllProjects || projects.contains(ndProject)) {
+                generateNamespaces(NS_PROJECT.ND4J, outputDir, "org.nd4j.linalg.factory");
+            }
+            if (forAllProjects || projects.contains(sdProject)) {
+                generateNamespaces(NS_PROJECT.SAMEDIFF, outputDir, "org.nd4j.autodiff.samediff");
+            }
+        } catch (Exception e) {
+            log.error(e.toString());
+        }
+    }
+}

codegen/op-codegen/src/main/java/org/nd4j/codegen/cli/PicoCliCodeGen.java

@@ -0,0 +1,182 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.cli;
+
+import com.beust.jcommander.*;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.nd4j.autodiff.samediff.SDVariable;
+import org.nd4j.codegen.Namespace;
+import org.nd4j.codegen.api.LossReduce;
+import org.nd4j.linalg.api.buffer.DataType;
+import picocli.CommandLine;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Planned CLI for generating classes
+ */
+@Slf4j
+public class PicoCliCodeGen {
+    private static final String relativePath = "nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/";
+    private static final String allProjects = "all";
+
+
+    @Parameter(names = "-dir", description = "Root directory of deeplearning4j mono repo")
+    private String repoRootDir;
+
+    @Parameter(names = "-docsdir", description = "Root directory for generated docs")
+    private String docsdir;
+
+    @Parameter(names = "-namespaces", description = "List of namespaces to generate, or 'ALL' to generate all namespaces", required = true)
+    private List<String> namespaces;
+
+
+
+    private void generateNamespaces() {
+
+        List<Namespace> usedNamespaces = new ArrayList<>();
+
+        for (String s : namespaces) {
+            if ("all".equalsIgnoreCase(s)) {
+                Collections.addAll(usedNamespaces, Namespace.values());
+                break;
+            }
+
+
+            CommandLine.Model.CommandSpec commandSpec = CommandLine.Model.CommandSpec.create();
+
+            int cnt = 0;
+            for (int i = 0; i < usedNamespaces.size(); ++i) {
+                Namespace ns = usedNamespaces.get(i);
+                CommandLine.Model.CommandSpec subCommand = CommandLine.Model.CommandSpec.create();
+                commandSpec.addSubcommand(ns.name(), subCommand);
+                ns.getNamespace().getOps().forEach(op -> {
+                    CommandLine.Model.CommandSpec commandSpec1 = CommandLine.Model.CommandSpec.create();
+                    subCommand.addSubcommand(op.name(), commandSpec1);
+                    op.inputs().forEach(input -> {
+                        //TODO: Add SDVariable converter for picocli and figure out where to put that converter
+                        commandSpec1.addOption(CommandLine.Model.OptionSpec.builder("--" + input.getName())
+                                .type(SDVariable.class)
+                                .required(true)
+                                .description(input.getDescription())
+                                .build());
+                    });
+
+                    op.getArgs().forEach(arg -> {
+                        CommandLine.Model.OptionSpec.Builder builder = CommandLine.Model.OptionSpec.builder("--" + arg.getName())
+                                .description(arg.getDescription());
+
+                        switch (arg.getType()) {
+                            case INT:
+                                builder.type(Integer.class);
+                                break;
+                            case BOOL:
+                                builder.type(Boolean.class);
+                                break;
+                            case ENUM:
+                                break;
+                            case LONG:
+                                builder.type(Long.class);
+                                break;
+                            case STRING:
+                                builder.type(String.class);
+                                break;
+                            case NDARRAY:
+                                break;
+                            case NUMERIC:
+                                break;
+                            case CONDITION:
+                                break;
+                            case DATA_TYPE:
+                                builder.type(DataType.class);
+                                break;
+                            case LOSS_REDUCE:
+                                builder.type(LossReduce.class);
+                                break;
+                            case FLOATING_POINT:
+                                break;
+                        }
+
+                        builder.required(arg.getDefaultValue() == null);
+
+                        if (arg.getDefaultValue() != null) {
+                            builder.defaultValue(arg.getDefaultValue().toString());
+                        }
+
+                        commandSpec1.addOption(builder.build());
+                    });
+
+
+                });
+                log.info("Starting generation of namespace: {}", ns);
+
+                ++cnt;
+            }
+
+
+            log.info("Complete - generated {} namespaces", cnt);
+        }
+    }
+
+
+    public static void main(String[] args) throws Exception {
+        new CLI().runMain(args);
+    }
+
+    public void runMain(String[] args) throws Exception {
+        JCommander.newBuilder()
+                .addObject(this)
+                .build()
+                .parse(args);
+
+        // Either root directory for source code generation or docs directory must be present. If root directory is
+        // absenbt - then it's "generate docs only" mode.
+        if (StringUtils.isEmpty(repoRootDir) && StringUtils.isEmpty(docsdir)) {
+            throw new IllegalStateException("Provide one or both of arguments : -dir, -docsdir");
+        }
+
+        File outputDir = null;
+        if (StringUtils.isNotEmpty(repoRootDir)) {
+            //First: Check root directory.
+            File dir = new File(repoRootDir);
+            if (!dir.exists() || !dir.isDirectory()) {
+                throw new IllegalStateException("Provided root directory does not exist (or not a directory): " + dir.getAbsolutePath());
+            }
+
+            outputDir = new File(dir, relativePath);
+            if (!outputDir.exists() || !dir.isDirectory()) {
+                throw new IllegalStateException("Expected output directory does not exist: " + outputDir.getAbsolutePath());
+            }
+        }
+
+        if(namespaces == null || namespaces.isEmpty() ) {
+            throw new IllegalStateException("No namespaces were provided");
+        }
+
+        generateNamespaces();
+
+    }
+}

codegen/op-codegen/src/main/java/org/nd4j/codegen/impl/cpp/CppGenerator.java

@@ -0,0 +1,155 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.impl.cpp;
+
+import org.apache.commons.io.FileUtils;
+import org.nd4j.codegen.api.*;
+import org.nd4j.codegen.api.generator.Generator;
+import org.nd4j.codegen.api.generator.GeneratorConfig;
+import org.nd4j.codegen.util.GenUtil;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A very simple, manual CPP generator
+ * As per Python, this could be implemented using a templating library such as freemarker
+ */
+public class CppGenerator implements Generator {
+    @Override
+    public Language language() {
+        return Language.CPP;
+    }
+
+    @Override
+    public void generateNamespaceNd4j(NamespaceOps namespace, GeneratorConfig config, File directory, String fileName) throws IOException {
+
+        StringBuilder sb = new StringBuilder();
+
+        sb.append("#include <NDArrayFactory.h>\n\n")
+                .append("namespace nd4j {\n");
+
+        append(4, sb, "namespace " + namespace.getName().toLowerCase());
+
+        List<Op> ops = new ArrayList<>();
+        for(Op o : namespace.getOps()){
+            if(o.isAbstract())
+                continue;
+            ops.add(o);
+        }
+
+        //TODO: handle includes
+
+        for(Op o : ops){
+            String s = generateFunction(o);
+            sb.append(GenUtil.addIndent(s, 8));
+            sb.append("\n");
+        }
+
+        append(4, sb, "}");
+        sb.append("}");
+
+        //TODO generate header also
+
+        String out = sb.toString();
+        File outFile = new File(directory, GenUtil.ensureFirstIsCap(namespace.getName()) + ".cpp");
+        FileUtils.writeStringToFile(outFile, out, StandardCharsets.UTF_8);
+    }
+
+    protected static void append(int indent, StringBuilder sb, String line){
+        sb.append(GenUtil.repeat(" ", indent))
+                .append(line)
+                .append("\n");
+    }
+
+    protected static String generateFunction(Op op){
+        StringBuilder sb = new StringBuilder();
+
+        List<Output> outputs = op.getOutputs();
+        boolean singleOut = outputs.size() == 1;
+        if(singleOut){
+            sb.append("NDArray* ");
+        } else {
+            throw new UnsupportedOperationException("Multi-output op generation not yet implemented");
+        }
+
+        sb.append(GenUtil.ensureFirstIsNotCap(op.getOpName())).append("(");
+
+        //Add inputs to signature
+        boolean firstArg = true;
+        if(op.getInputs() != null){
+            for(Input i : op.getInputs()){
+                if(!firstArg)
+                    sb.append(", ");
+
+                sb.append("NDArray* ").append(i.getName());
+
+                firstArg = false;
+            }
+        }
+
+
+        //Add arguments and default args to signature
+        sb.append("):\n");
+
+
+        sb.append("    Context c(1);\n");
+        int j=0;
+        for(Input i : op.getInputs()){
+            sb.append("    c.setInputArray(").append(j++).append(", ").append(i.getName()).append(");\n");
+        }
+
+        sb.append("\n    //TODO: args\n\n");
+
+
+        sb.append("    nd4j::ops::").append(op.getLibnd4jOpName()).append(" op;\n");
+
+        sb.append("    ShapeList shapeList({");
+        j = 0;
+        for(Input i : op.getInputs()){
+            if(j > 0)
+                sb.append(",");
+            sb.append(i.getName());
+            j++;
+        }
+
+        sb.append("});\n\n")
+                .append("    auto outShape = op.calculateOutputShape(&shapeList, c);\n");
+
+        sb.append("    auto out = nullptr;  //TODO\n\n")
+                .append("    op.exec(c);\n")
+                .append("    delete shapes;\n");
+
+        sb.append("    return out;\n")
+                .append("}\n");
+
+
+        return sb.toString();
+    }
+
+    @Override
+    public void generateNamespaceSameDiff(NamespaceOps namespace, GeneratorConfig config, File directory, String fileName) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+}

codegen/op-codegen/src/main/java/org/nd4j/codegen/impl/java/DocsGenerator.java

@@ -0,0 +1,301 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.impl.java;
+
+import com.squareup.javapoet.MethodSpec;
+import com.squareup.javapoet.TypeName;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.nd4j.codegen.api.*;
+import org.nd4j.codegen.api.doc.DocSection;
+import org.nd4j.codegen.api.doc.DocTokens;
+import org.nd4j.codegen.util.GenUtil;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+
+import static org.nd4j.codegen.impl.java.Nd4jNamespaceGenerator.exactlyOne;
+
+public class DocsGenerator {
+
+    // Markdown marker for start-end of code section
+    private static final String MD_CODE = "```";
+    // Javadoc constants which should be dropped or replaced for markdown generation
+    private static final String JD_CODE = "{@code ";
+    private static final String JD_CODE_END = "}";
+    private static final String JD_INPUT_TYPE = "%INPUT_TYPE%";
+
+    public static class JavaDocToMDAdapter {
+        private String current;
+
+        public JavaDocToMDAdapter(String original) {
+            this.current = original;
+        }
+
+        public JavaDocToMDAdapter filter(String pattern, String replaceWith) {
+            String result =  StringUtils.replace(current, pattern, replaceWith);
+            this.current = result;
+            return this;
+        }
+
+        @Override
+        public String toString() {
+            return current;
+        }
+    }
+
+    private static String generateMethodText(Op op, Signature s, boolean isSameDiff, boolean isLoss, boolean withName) {
+        StringBuilder sb = new StringBuilder();
+        MethodSpec.Builder c = MethodSpec.methodBuilder(GenUtil.ensureFirstIsNotCap(op.getOpName()));
+        List<Parameter> params = s.getParameters();
+        List<Output> outs = op.getOutputs();
+        String retType = "void";
+
+        if (outs.size() == 1) {
+            retType = isSameDiff ? "SDVariable" : "INDArray";
+        }
+        else if (outs.size() >= 1) {
+            retType = isSameDiff ? "SDVariable[]" : "INDArray[]";
+        }
+        sb.append(retType).append(" ").append(op.getOpName()).append("(");
+        boolean first = true;
+        for (Parameter param : params) {
+            if (param instanceof Arg) {
+                Arg arg = (Arg) param;
+                if (!first)
+                    sb.append(", ");
+                else if (withName)
+                    sb.append("String name, ");
+                String className;
+                if(arg.getType() == DataType.ENUM) {
+                    className = GenUtil.ensureFirstIsCap(arg.name());
+                } else {
+                    TypeName tu = Nd4jNamespaceGenerator.getArgType(arg);
+                    className = tu.toString();
+                }
+                if(className.contains(".")){
+                    className = className.substring(className.lastIndexOf('.')+1);
+                }
+                sb.append(className).append(" ").append(arg.name());
+                first = false;
+            }
+            else if (param instanceof Input) {
+                Input arg = (Input) param;
+                if (!first)
+                    sb.append(", ");
+                else if (withName)
+                    sb.append("String name, ");
+                sb.append(isSameDiff ? "SDVariable " : "INDArray ").append(arg.name());
+                first = false;
+            } else if(param instanceof Config){
+                if(!first)
+                    sb.append(", ");
+                Config conf = (Config)param;
+                String name = conf.getName();
+                sb.append(name).append(" ").append(GenUtil.ensureFirstIsNotCap(name));
+            }
+        }
+        sb.append(")");
+        return sb.toString();
+    }
+
+    private static StringBuilder buildDocSectionText(List<DocSection> docSections) {
+        StringBuilder sb = new StringBuilder();
+        for (DocSection ds : docSections) {
+            //if(ds.applies(Language.JAVA, CodeComponent.OP_CREATOR)){
+            String text = ds.getText();
+            String[] lines = text.split("\n");
+            for (int i = 0; i < lines.length; i++) {
+                if (!lines[i].endsWith("<br>")) {
+                    String filteredLine = new JavaDocToMDAdapter(lines[i])
+                            .filter(JD_CODE, "`")
+                            .filter(JD_CODE_END, "`")
+                            .filter(JD_INPUT_TYPE, "INDArray").toString();
+
+                    lines[i] = filteredLine + System.lineSeparator();
+                }
+            }
+            text = String.join("\n", lines);
+            sb.append(text).append(System.lineSeparator());
+            //}
+        }
+        return sb;
+    }
+
+    public static void generateDocs(int namespaceNum, NamespaceOps namespace, String docsDirectory, String basePackage) throws IOException {
+        File outputDirectory = new File(docsDirectory);
+        StringBuilder sb = new StringBuilder();
+        String headerName = namespace.getName();
+        if(headerName.startsWith("SD"))
+            headerName = headerName.substring(2);
+
+        // File Header for Gitbook
+        sb.append("---").append(System.lineSeparator());
+        sb.append("title: ").append(headerName).append(System.lineSeparator());
+        sb.append("short_title: ").append(headerName).append(System.lineSeparator());
+        sb.append("description: ").append(System.lineSeparator());
+        sb.append("category: Operations").append(System.lineSeparator());
+        sb.append("weight: ").append(namespaceNum * 10).append(System.lineSeparator());
+        sb.append("---").append(System.lineSeparator());
+
+        List<Op> ops = namespace.getOps();
+
+        ops.sort(Comparator.comparing(Op::getOpName));
+
+        if (ops.size() > 0)
+            sb.append("# Operation classes").append(System.lineSeparator());
+        for (Op op : ops) {
+            sb.append("## ").append(op.getOpName()).append(System.lineSeparator());
+            List<DocSection> doc = op.getDoc();
+            if(!doc.isEmpty()) {
+                boolean first = true;
+                StringBuilder ndSignatures = new StringBuilder();
+                StringBuilder sdSignatures = new StringBuilder();
+                StringBuilder sdNameSignatures = new StringBuilder();
+                for(Signature s : op.getSignatures()) {
+                    if (first) {
+                        Language lang = doc.get(0).getLanguage();
+                        sb.append(MD_CODE).append(lang.equals(Language.ANY) ? Language.JAVA : lang).append(System.lineSeparator());
+                        first = false;
+                    }
+                    String ndCode = generateMethodText(op, s, false, false, false);
+                    ndSignatures.append(ndCode).append(System.lineSeparator());
+                    String sdCode = generateMethodText(op, s, true, false, false);
+                    sdSignatures.append(sdCode).append(System.lineSeparator());
+                    String withNameCode = generateMethodText(op, s, true, false, true);
+                    sdNameSignatures.append(withNameCode).append(System.lineSeparator());
+                }
+                sb.append(ndSignatures.toString());
+                sb.append(System.lineSeparator()); // New line between INDArray and SDVariable signatures
+
+                sb.append(sdSignatures.toString());
+                sb.append(sdNameSignatures.toString());
+
+                sb.append(MD_CODE).append(System.lineSeparator());
+                StringBuilder tsb = buildDocSectionText(doc);
+                sb.append(tsb.toString());
+                List<Signature> l = op.getSignatures();
+                Set<Parameter> alreadySeen = new HashSet<>();
+                for(Signature s : l) {
+                    List<Parameter> params = s.getParameters();
+                    for (Parameter p : params) {
+                        if(alreadySeen.contains(p)) continue;
+                        alreadySeen.add(p);
+                        if(p instanceof Input){
+                            Input i = (Input)p;
+                            sb.append("* **").append(i.getName()).append("** ").append(" (").append(i.getType()).append(") - ").append(i.getDescription() == null ? "" : DocTokens.processDocText(i.getDescription(),
+                                    op, DocTokens.GenerationType.ND4J)).append(System.lineSeparator());
+                        }
+                        else if (p instanceof Config) {
+                            Config c = (Config)p;
+                            sb.append("* **").append(c.getName()).append("** - see ").append("[").append(c.getName()).append("]").append("(#").append(toAnchor(c.getName())).append(")").append(System.lineSeparator());
+                        }
+                        else if(p instanceof Arg) {
+                            Arg arg = (Arg) p;
+                            final Count count = arg.getCount();
+                            if (count == null || count.equals(exactlyOne)) {
+                                sb.append("* **").append(arg.getName()).append("** - ").append(arg.getDescription() == null ? "" : DocTokens.processDocText(arg.getDescription(),
+                                        op, DocTokens.GenerationType.ND4J));    //.append(System.lineSeparator());
+                            } else {
+                                sb.append("* **").append(arg.getName()).append("** - ").append(arg.getDescription() == null ? "" : DocTokens.processDocText(arg.getDescription(),
+                                        op, DocTokens.GenerationType.ND4J)).append(" (Size: ").append(count.toString()).append(")");    //.append(System.lineSeparator());
+                            }
+
+                            Object defaultValue = arg.defaultValue();
+                            if(defaultValue != null){
+                                sb.append(" - default = ").append(formatDefaultValue(defaultValue));
+                            }
+
+                            sb.append(System.lineSeparator());
+                        }
+                    }
+                }
+                sb.append(System.lineSeparator());
+            }
+        }
+
+
+        if (namespace.getConfigs().size() > 0)
+            sb.append("# Configuration Classes").append(System.lineSeparator());
+        for (Config config : namespace.getConfigs()) {
+            sb.append("## ").append(config.getName()).append(System.lineSeparator());
+            for (Input i : config.getInputs()) {
+                sb.append("* **").append(i.getName()).append("**- ").append(i.getDescription()).append(" (").append(i.getType()).append(" type)");
+                if (i.hasDefaultValue() && (i.defaultValue() != null))
+                    sb.append(" Default value:").append(formatDefaultValue(i.defaultValue())).append(System.lineSeparator());
+                else
+                    sb.append(System.lineSeparator());
+            }
+            for (Arg arg : config.getArgs()) {
+                sb.append("* **").append(arg.getName()).append("** ").append("(").append(arg.getType()).append(") - ").append(arg.getDescription());
+                if (arg.hasDefaultValue() && (arg.defaultValue() != null))
+                    sb.append(" - default = ").append(formatDefaultValue(arg.defaultValue())).append(System.lineSeparator());
+                else
+                    sb.append(System.lineSeparator());
+            }
+            StringBuilder tsb = buildDocSectionText(config.getDoc());
+            sb.append(tsb.toString());
+            sb.append(System.lineSeparator());
+            for (Op op : ops) {
+                if (op.getConfigs().contains(config)) {
+                    sb.append("Used in these ops: " + System.lineSeparator());
+                    break;
+                }
+            }
+            ops.stream().filter(op -> op.getConfigs().contains(config)).forEach(op ->
+                       sb.append("[").append(op.getOpName()).append("]").append("(#").append(toAnchor(op.getOpName())).append(")").
+                       append(System.lineSeparator()));
+
+        }
+        File outFile = new File(outputDirectory + "/operation-namespaces", "/" + namespace.getName().toLowerCase() + ".md");
+        FileUtils.writeStringToFile(outFile, sb.toString(), StandardCharsets.UTF_8);
+    }
+
+    private static String formatDefaultValue(Object v){
+        if(v == null){ return "null"; }
+        else if(v instanceof int[]){ return Arrays.toString((int[]) v); }
+        else if(v instanceof long[]){ return Arrays.toString((long[]) v); }
+        else if(v instanceof float[]){ return Arrays.toString((float[]) v); }
+        else if(v instanceof double[]){ return Arrays.toString((double[]) v); }
+        else if(v instanceof boolean[]){ return Arrays.toString((boolean[]) v); }
+        else if(v instanceof Input){ return ((Input)v).getName(); }
+        else if(v instanceof org.nd4j.linalg.api.buffer.DataType){ return "DataType." + v; }
+        else if(v instanceof LossReduce || v instanceof org.nd4j.autodiff.loss.LossReduce){ return "LossReduce." + v; }
+        else return v.toString();
+    }
+
+    private static String toAnchor(String name){
+        int[] codepoints = name.toLowerCase().codePoints().toArray();
+        int type = Character.getType(codepoints[0]);
+        StringBuilder anchor = new StringBuilder(new String(Character.toChars(codepoints[0])));
+        for (int i = 1; i < codepoints.length; i++) {
+            int curType = Character.getType(codepoints[i]);
+            if(curType != type){
+                anchor.append("-");
+            }
+            type = curType;
+            anchor.append(new String(Character.toChars(codepoints[i])));
+        }
+        return anchor.toString();
+    }
+}

codegen/op-codegen/src/main/java/org/nd4j/codegen/impl/java/JavaPoetGenerator.java

@@ -0,0 +1,48 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.impl.java;
+import org.apache.commons.lang3.StringUtils;
+import org.nd4j.codegen.api.Language;
+import org.nd4j.codegen.api.NamespaceOps;
+import org.nd4j.codegen.api.generator.Generator;
+import org.nd4j.codegen.api.generator.GeneratorConfig;
+
+import java.io.File;
+
+public class JavaPoetGenerator implements Generator {
+
+
+    @Override
+    public Language language() {
+        return Language.JAVA;
+    }
+
+    @Override
+    public void generateNamespaceNd4j(NamespaceOps namespace, GeneratorConfig config, File directory, String className) throws java.io.IOException {
+        Nd4jNamespaceGenerator.generate(namespace, config, directory, className, "org.nd4j.linalg.factory", StringUtils.EMPTY);
+    }
+
+    @Override
+    public void generateNamespaceSameDiff(NamespaceOps namespace, GeneratorConfig config, File directory, String className) throws java.io.IOException {
+        //throw new UnsupportedOperationException("Not yet implemented");
+        Nd4jNamespaceGenerator.generate(namespace, config, directory, className, "org.nd4j.autodiff.samediff", StringUtils.EMPTY);
+    }
+}

codegen/op-codegen/src/main/java/org/nd4j/codegen/impl/java/Nd4jNamespaceGenerator.java

@@ -0,0 +1,989 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.impl.java;
+
+import com.squareup.javapoet.*;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.jetbrains.annotations.NotNull;
+import org.nd4j.autodiff.samediff.SDVariable;
+import org.nd4j.autodiff.samediff.SameDiff;
+import org.nd4j.autodiff.samediff.ops.SDOps;
+import org.nd4j.autodiff.samediff.ops.SDValidation;
+import org.nd4j.codegen.api.*;
+import org.nd4j.codegen.api.doc.DocSection;
+import org.nd4j.codegen.api.doc.DocTokens;
+import org.nd4j.codegen.api.generator.ConstraintCodeGenerator;
+import org.nd4j.codegen.api.generator.GeneratorConfig;
+import org.nd4j.codegen.util.GenUtil;
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.factory.NDValidation;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.indexing.conditions.Condition;
+
+import javax.lang.model.element.Modifier;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class Nd4jNamespaceGenerator {
+    private static Map<DataType, Class<?>> typeMapping = new HashMap<>();
+    private static Map<DataType, String> validationMapping = new HashMap<>();
+    private static Map<Arg, TypeName> enumMapping = new HashMap<>();
+    private static Map<Config, TypeName> configMapping = new HashMap<>();
+    public static Count exactlyOne = new Exactly(1);
+    private static String copyright =
+            "/*\n" +
+                    " *  ******************************************************************************\n" +
+                    " *  *\n" +
+                    " *  *\n" +
+                    " *  * This program and the accompanying materials are made available under the\n" +
+                    " *  * terms of the Apache License, Version 2.0 which is available at\n" +
+                    " *  * https://www.apache.org/licenses/LICENSE-2.0.\n" +
+                    " *  *\n" +
+                    " *  *  See the NOTICE file distributed with this work for additional\n" +
+                    " *  *  information regarding copyright ownership.\n" +
+                    " *  * Unless required by applicable law or agreed to in writing, software\n" +
+                    " *  * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n" +
+                    " *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n" +
+                    " *  * License for the specific language governing permissions and limitations\n" +
+                    " *  * under the License.\n" +
+                    " *  *\n" +
+                    " *  * SPDX-License-Identifier: Apache-2.0\n" +
+                    " *  *****************************************************************************\n" +
+                    " */\n";
+    private static String codeGenWarning =
+            "\n//================== GENERATED CODE - DO NOT MODIFY THIS FILE ==================\n\n";
+
+    static {
+        typeMapping.put(DataType.BOOL, boolean.class);
+        typeMapping.put(DataType.FLOATING_POINT, double.class);
+        typeMapping.put(DataType.NUMERIC, double.class);
+        typeMapping.put(DataType.INT, int.class);
+        typeMapping.put(DataType.LONG, long.class);
+        typeMapping.put(DataType.DATA_TYPE, org.nd4j.linalg.api.buffer.DataType.class);
+        typeMapping.put(DataType.LOSS_REDUCE, org.nd4j.autodiff.loss.LossReduce.class);
+        typeMapping.put(DataType.CONDITION, Condition.class);
+        typeMapping.put(DataType.STRING, String.class);
+
+        validationMapping.put(DataType.BOOL, "validateBool");
+        validationMapping.put(DataType.FLOATING_POINT, "validateFloatingPoint");
+        validationMapping.put(DataType.NUMERIC, "validateNumerical");
+        validationMapping.put(DataType.INT, "validateInteger");
+        validationMapping.put(DataType.LONG, "validateInteger");
+    }
+
+    private static ConstraintCodeGenerator constraintCodeGenerator = new JavaConstraintCodeGenerator();
+
+    private Nd4jNamespaceGenerator() { }
+
+    public static void generate(NamespaceOps namespace, GeneratorConfig config, File outputDirectory, String className,
+                                String basePackage, String docsDirectory) throws IOException {
+        //String basePackage = "org.nd4j.linalg.factory";
+
+        generateEnums(outputDirectory, basePackage);
+        generateConfigs(outputDirectory, basePackage);
+        try {
+            generateOpFactory(namespace, outputDirectory, className, basePackage, StringUtils.EMPTY);
+        }
+        catch (Exception e) {
+            log.error(e.toString());
+        }
+    }
+
+    public static void generate(NamespaceOps namespace, GeneratorConfig config, File outputDirectory, String className,
+                                String basePackage, String parentClass, String docsDirectory) throws IOException {
+        //String basePackage = "org.nd4j.linalg.factory";
+
+        generateEnums(outputDirectory, basePackage);
+        generateConfigs(outputDirectory, basePackage);
+        try {
+            generateOpFactory(namespace, outputDirectory, className, basePackage, parentClass);
+        }
+        catch (Exception e) {
+            log.error(e.toString());
+        }
+    }
+
+    private static void generateOpFactory(NamespaceOps namespace, File outputDirectory, String className, String basePackage,
+                                          String parentClass) throws IOException, ClassNotFoundException {
+        boolean isBaseSameDiff = StringUtils.equals("SDBaseOps", className);
+        boolean isSameDiff = StringUtils.isNotEmpty(parentClass);
+        boolean isLoss = StringUtils.equals("SDLoss", className);
+
+        TypeSpec.Builder builder = !isSameDiff || isBaseSameDiff ?
+                 TypeSpec.classBuilder(className)
+                    .addModifiers(Modifier.PUBLIC) :
+
+                 TypeSpec.classBuilder(className)
+                    .superclass(Class.forName(parentClass))
+                    .addModifiers(Modifier.PUBLIC);
+
+        if (isSameDiff && !isBaseSameDiff) {
+            addSameDiffConstructor(builder);
+        }
+        else if (isBaseSameDiff) {
+            builder.addField(TypeName.get(SameDiff.class), "sd", Modifier.PROTECTED);
+            addBaseSameDiffConstructor(builder);
+        }
+        else
+            addDefaultConstructor(builder);
+
+        //Add ops
+        namespace.getOps()
+                .stream()
+                .filter(it -> !it.isAbstract())
+                .sorted(Comparator.comparing(Op::getOpName))
+                .forEachOrdered(o -> generateMethods(builder, o, isSameDiff, isLoss));
+
+
+        TypeSpec ts = builder.build();
+
+        final String opsPackage = basePackage + ".ops";
+        JavaFile jf = StringUtils.isEmpty(parentClass) ?
+
+                JavaFile.builder(opsPackage, ts)
+                .addStaticImport(NDValidation.class, "isSameType")
+                .build() :
+
+                JavaFile.builder(opsPackage, ts)
+                        .addStaticImport(SDValidation.class, "isSameType")
+                        .build();
+
+        StringBuilder sb = new StringBuilder();
+        sb.append(copyright);
+        sb.append(codeGenWarning);
+        jf.writeTo(sb);
+
+        File outFile = new File(outputDirectory, packageToDirectory(opsPackage) + "/" + className + ".java");
+        FileUtils.writeStringToFile(outFile, sb.toString(), StandardCharsets.UTF_8);
+    }
+
+    private static String packageToDirectory(String packageName){
+        return packageName.replace(".", File.separator);
+    }
+
+    private static void addDefaultConstructor(TypeSpec.Builder builder) {
+        //Add private no-arg constructor
+        MethodSpec noArg = MethodSpec.constructorBuilder()
+                .addModifiers(Modifier.PUBLIC)
+                .build();
+
+        builder.addMethod(noArg);
+    }
+
+    private static void addBaseSameDiffConstructor(TypeSpec.Builder builder) {
+
+        MethodSpec ctor = MethodSpec.constructorBuilder()
+                .addModifiers(Modifier.PUBLIC)
+                .addParameter(SameDiff.class, "sameDiff")
+                .addStatement("this.sd = sameDiff")
+                .build();
+
+        builder.addMethod(ctor);
+    }
+
+    private static void addSameDiffConstructor(TypeSpec.Builder builder) {
+        MethodSpec ctor = MethodSpec.constructorBuilder()
+                .addModifiers(Modifier.PUBLIC)
+                .addParameter(SameDiff.class, "sameDiff")
+                .addStatement("super(sameDiff)")
+                .build();
+
+        builder.addMethod(ctor);
+    }
+
+    private static void generateMethods(TypeSpec.Builder builder, Op op, boolean isSameDiff, boolean isLoss ){
+        List<Signature> l = op.getSignatures();
+        for(Signature s : l){
+            builder.addMethod(signatureCreatorMethod(op, s, isSameDiff, false, isLoss));
+            if (isSameDiff)
+                builder.addMethod(signatureCreatorMethod(op, s, true, true, isLoss));
+        }
+    }
+
+    private static MethodSpec signatureCreatorMethod(Op op, Signature s, boolean isSameDiff, boolean withName,
+                                                     boolean isLoss){
+        MethodSpec.Builder c = MethodSpec.methodBuilder(GenUtil.ensureFirstIsNotCap(op.getOpName()))
+                .addModifiers(Modifier.PUBLIC);
+        enableVarargsOnLastArg(c, op, s);
+
+        buildJavaDoc(op, s, c, withName);
+        List<String> inNames = buildParameters(c, op, s, isSameDiff, withName);
+        buildConstraints(c, op.getConstraints());
+        buildExecution(c, op, inNames, isSameDiff, withName, isLoss);
+
+        return c.build();
+    }
+
+    private static void buildJavaDoc(Op op, Signature s, MethodSpec.Builder c, boolean withName) {
+        //Method javadoc:
+        List<DocSection> doc = op.getDoc();
+        if(!doc.isEmpty()){
+            for(DocSection ds : doc){
+                if(ds.applies(Language.JAVA, CodeComponent.OP_CREATOR)){
+                    String text = DocTokens.processDocText(ds.getText(), op, DocTokens.GenerationType.ND4J);
+                    //Add <br> tags at the end of each line, where none already exists
+                    String[] lines = text.split("\n");
+                    for( int i = 0; i < lines.length; i++) {
+                        if(!lines[i].endsWith("<br>")){
+                            lines[i] = lines[i] + "<br>";
+                        }
+                    }
+                    text = String.join("\n", lines);
+                    c.addJavadoc(text + "\n\n");
+                }
+            }
+        }
+
+
+        // Document Constraints:
+        //TODO what if constraint is on default value arg/s - no point specifying them here...
+        final List<Constraint> constraints = op.getConstraints();
+        if(!constraints.isEmpty()){
+            c.addJavadoc("Inputs must satisfy the following constraints: <br>\n");
+            for (Constraint constraint : constraints) {
+                c.addJavadoc(constraint.getMessage() +": " + constraintCodeGenerator.generateExpression(constraint.getCheck()) + "<br>\n");
+            }
+
+            c.addJavadoc("\n");
+        }
+        if (withName) {
+            if (op.getOutputs().size() == 1 && !op.getOutputs().get(0).getMultiOutput())
+                c.addJavadoc("@param name name May be null. Name for the output variable\n");
+            else
+                c.addJavadoc("@param names names May be null. Arrays of names for the output variables.\n");
+        }
+        List<Parameter> params = s.getParameters();
+        if(!params.isEmpty()){
+            for(Parameter p : params){
+                if(p instanceof Input){
+                    Input i = (Input)p;
+                    c.addJavadoc("@param " + i.getName() + " " + (i.getDescription() == null ? "" : DocTokens.processDocText(i.getDescription(), op, DocTokens.GenerationType.ND4J)) + " (" + i.getType() + " type)\n");
+                } else if(p instanceof Arg) {
+                    Arg arg = (Arg) p;
+                    final Count count = arg.getCount();
+                    if (count == null || count.equals(exactlyOne)) {
+                        c.addJavadoc("@param " + arg.getName() + " " + (arg.getDescription() == null ? "" : DocTokens.processDocText(arg.getDescription(), op, DocTokens.GenerationType.ND4J)) + "\n");
+                    } else {
+                        c.addJavadoc("@param " + arg.getName() + " " + (arg.getDescription() == null ? "" : DocTokens.processDocText(arg.getDescription(), op, DocTokens.GenerationType.ND4J)) + " (Size: " + count.toString() + ")\n");
+                    }
+                } else if(p instanceof Config){
+                    Config config = (Config) p;
+                    c.addJavadoc("@param " + config.getName() + " Configuration Object\n");
+                } else {
+                    throw new RuntimeException("Unknown parameter type: " + p + " - " + p.getClass() + " - op = " + op.getOpName());
+                }
+            }
+
+
+        }
+
+        //Outputs:
+        List<Output> outputs = op.getOutputs();
+        if(!outputs.isEmpty()){
+            if(outputs.size() == 1 && !outputs.get(0).getMultiOutput()){
+                Output o = outputs.get(0);
+                c.addJavadoc("@return " + o.getName() + " " + (o.getDescription() == null ? "" : DocTokens.processDocText(o.getDescription(), op, DocTokens.GenerationType.ND4J)) + " (" + o.getType() + " type)\n");
+            } else {
+                //throw new UnsupportedOperationException("Javadoc for multi-output ops not yet implemented");
+                log.error("Javadoc for multi-output ops not yet implemented");
+            }
+        }
+    }
+
+    private static List<String> buildParameters(MethodSpec.Builder c, Op op, Signature s, boolean isSameDiff, boolean withName) {
+        List<String> inNames = new ArrayList<>();
+
+        List<Parameter> params = s.getParameters();
+
+        if(op.getArgsFirst()){
+            //Assuming sort is stable (doesn't change order of equal elements)
+            params.sort((p1,p2) -> Boolean.compare(p1 instanceof Input, p2 instanceof Input));
+        }
+
+        if (withName) {
+            if (op.getOutputs().size() == 1 && !op.getOutputs().get(0).getMultiOutput())
+                c.addParameter(String.class, "name");
+            else
+                c.addParameter(String[].class, "names");
+        }
+        if(!params.isEmpty()){
+            int pCount = 0;
+            for(Parameter p : params){
+                pCount++;
+                boolean isLast = pCount == params.size();
+                if(p instanceof Input){
+                    Input i = (Input)p;
+                    final String inputName = i.getName();
+                    inNames.add(inputName);
+
+                    final Count count = i.getCount();
+                    if(count == null || count.equals(exactlyOne)) {
+                        //Single input
+                        if (isSameDiff)
+                            c.addParameter(SDVariable.class, inputName);
+                        else
+                            c.addParameter(INDArray.class, inputName);
+                    } else {
+                        //Array input
+                        if (isSameDiff)
+                            c.addParameter(SDVariable[].class, inputName).varargs(isLast);
+                        else
+                            c.addParameter(INDArray[].class, inputName).varargs(isLast);
+                    }
+                    // Check for parameter types
+                    final DataType paramType = i.getType();
+                    String validationName = validationMapping.get(paramType);
+                    if(validationName != null) {
+                        c.addStatement(CodeBlock.of("$T.$L($S, $S, $L)", isSameDiff ? SDValidation.class : NDValidation.class, validationName, op.getOpName(), inputName, inputName));
+                    }
+                    checkParameterCount(c, count, inputName);
+                } else if(p instanceof Arg){
+                    Arg arg = (Arg)p;
+                    final String argName = arg.getName();
+                    if(argName.isEmpty()){
+                        throw new IllegalStateException("Got null argument name for op " + op.getOpName());
+                    }
+                    inNames.add(argName);
+
+
+                    final Count count = arg.getCount();
+                    TypeName type = getArgType(arg);
+                    if(type == null){
+                        throw new IllegalStateException("No type mapping has been specified for type " + arg.getType() + " (op=" + op.getOpName() + ", arg=" + arg.getName() + ")" );
+                    }
+                    c.addParameter(type, argName);
+
+                    checkParameterCount(c, count, argName);
+                } else if(p instanceof Config) {
+                    Config config = (Config) p;
+                    final String configName = config.getName();
+                    inNames.add(configName);
+                    c.addParameter(configMapping.get(config), config.name());
+                } else {
+                    throw new IllegalStateException("Unknown parameter type: " + p + " - " + p.getClass());
+                }
+
+            }
+        }
+
+        return inNames;
+    }
+
+    public static TypeName getArgType(Arg arg) {
+        DataType argType = arg.getType();
+        Count count = arg.getCount();
+        TypeName type;
+        if(argType == DataType.ENUM){
+            type = enumMapping.get(arg);
+            if(type == null){
+                throw new IllegalStateException(arg + " is using an unregistered ENUM. This is probably a bug.");
+            }
+        }else{
+            if(!typeMapping.containsKey(argType)){
+                return null;
+            }
+            type = TypeName.get(typeMapping.get(argType));
+        }
+
+        if (!(count == null || count.equals(exactlyOne))) {
+            // array Arg
+            type = ArrayTypeName.of(type);
+        }
+        return type;
+    }
+
+    private static void buildConstraints(MethodSpec.Builder c, List<Constraint> constraints) {
+        if(constraints.isEmpty())
+            return;
+
+        //TODO not all contsraints apply to all signatures?
+
+        // Don't materialize the Backend Constraints
+        for (Constraint constraint : constraints.stream().filter(it -> !(it instanceof BackendConstraint)).collect(Collectors.toList())) {
+            c.addStatement(CodeBlock.of("$T.checkArgument($L, $S)", Preconditions.class, constraintCodeGenerator.generateExpression(constraint.getCheck()), constraint.getMessage()));
+        }
+    }
+
+    private static void buildExecution(MethodSpec.Builder c, Op op, List<String> inNames, boolean isSameDiff,
+                                       boolean withName, boolean isLoss) {
+        boolean singleOut = op.getOutputs().size() == 1 && !op.getOutputs().get(0).getMultiOutput();
+        if(singleOut){
+            if (isSameDiff)
+                c.returns(SDVariable.class);
+            else
+                c.returns(INDArray.class);
+        } else {
+            if (isSameDiff)
+                c.returns(SDVariable[].class);
+            else
+                c.returns(INDArray[].class);
+        }
+
+        // We have to pass all parameters, always. But not all signatures will be taking all parameters.
+        // inNames tells us which parameters this signatures has. For all others we want to pass default values
+        List<String> parameters = op.allParameters().stream().sorted(
+                (p1,p2) -> {
+                    if (p1.isVararg()) return 1;
+                    else if (p2.isVararg()) return -1;
+                    return 0;
+                }
+            ).map(it -> {
+            if(inNames.contains(it.name())){
+                return it.name();
+            }else{
+                if(!it.hasDefaultValue()) throw new IllegalStateException("The parameter "+it.name()+" has no default value, but is also not part of "+inNames.toString());
+                return anyToCode(it, it.defaultValue());
+            }
+        }).collect(Collectors.toList());
+
+        //Op execution:
+        StringBuilder sb = new StringBuilder();
+        if (isSameDiff) {
+            if (withName) {
+                if (singleOut)
+                    sb.append("SDVariable out = ");
+                else
+                    sb.append("SDVariable[] out = ");
+
+                sb.append(" new ")
+                        .append(op.getJavaPackage())
+                        .append(".")
+                        .append(op.getJavaOpClass() == null ? GenUtil.ensureFirstIsCap(op.getOpName()) : op.getJavaOpClass())
+                        .append("(sd,")
+                        .append(String.join(", ", parameters))
+                        .append(")");
+
+                if (singleOut)
+                    sb.append(".outputVariable()");
+                else
+                    sb.append(".outputVariables()");
+
+                c.addStatement(sb.toString());
+                if (isLoss)
+                    c.addStatement("out.markAsLoss()");
+
+                if (singleOut)
+                    c.addStatement("return sd.updateVariableNameAndReference(out, name)");
+                else
+                    c.addStatement("return sd.updateVariableNamesAndReferences(out, names)");
+            }
+            else {
+                if (isLoss) {
+                    sb.append("SDVariable out = new ")
+                            .append(op.getJavaPackage())
+                            .append(".")
+                            .append(op.getJavaOpClass() == null ? GenUtil.ensureFirstIsCap(op.getOpName()) : op.getJavaOpClass())
+                            .append("(sd,")
+                            .append(String.join(", ", parameters))
+                            .append(")");
+                }
+                else {
+                    sb.append("return new ")
+                            .append(op.getJavaPackage())
+                            .append(".")
+                            .append(op.getJavaOpClass() == null ? GenUtil.ensureFirstIsCap(op.getOpName()) : op.getJavaOpClass())
+                            .append("(sd,")
+                            .append(String.join(", ", parameters))
+                            .append(")");
+                }
+                    //if (!op.getLegacy()) {
+                    if (singleOut)
+                        sb.append(".outputVariable()");
+                    else
+                        sb.append(".outputVariables()");
+                    //}
+                c.addStatement(sb.toString());
+                if (isLoss) {
+                    c.addStatement("out.markAsLoss()");
+                    c.addStatement("return out");
+                }
+            }
+        }
+         else{
+            sb.append("return $T.exec(new ")
+                    .append(op.getJavaPackage())
+                    .append(".")
+                    .append(op.getJavaOpClass() == null ? GenUtil.ensureFirstIsCap(op.getOpName()) : op.getJavaOpClass())
+                    .append("(")
+                    .append(String.join(", ", parameters))
+                    .append("))");
+            if (!op.getLegacy() && singleOut)        //Note: legacy ops Nd4j.exec(Op) returns INDArray; Nd4j.exec(CustomOp) returns INDArray[]
+                sb.append("[0]");
+
+            c.addStatement(sb.toString(), Nd4j.class);
+        }
+    }
+
+    private static void enableVarargsOnLastArg(MethodSpec.Builder c, Op op, Signature s) {
+        List<Parameter> p = s.getParameters();
+        if(!p.isEmpty()){
+            Parameter lastP = p.get(p.size() - 1);
+            if (lastP instanceof Arg) {
+                Arg arg = (Arg) lastP;
+                final Count count = arg.getCount();
+                if (count != null && !count.equals(exactlyOne)) {
+                    c.varargs(true);
+                }
+            }
+        }
+    }
+
+    private static String countToJava(Count count,String paramName) {
+        final String paramLength = paramName + ".length";
+        if(count instanceof Exactly){
+            return paramLength + " == " + ((Exactly) count).getCount();
+        }else if(count instanceof AtLeast){
+            return paramLength + " >= " + ((AtLeast) count).getMin();
+        }else if(count instanceof AtMost){
+            return paramLength + " <= "+ ((AtMost) count).getMax();
+        }else if(count instanceof Range){
+            return ((Range) count).getFrom() + " <= " + paramLength + " && " + paramLength + " <= " + ((Range) count).getTo();
+        }else{
+            throw new IllegalArgumentException("Can not deal with Count of type " + count.getClass().getName());
+        }
+    }
+
+    private static void checkParameterCount(MethodSpec.Builder c, Count count, String paramName) {
+        // Check for parameter counts
+        if(count != null && !count.equals(exactlyOne)){
+            final String errorMessage = paramName + " has incorrect size/length. Expected: " + countToJava(count, paramName) + ", got %s";
+            if(count instanceof Exactly){
+                c.addStatement(CodeBlock.of("$T.checkArgument($L.length == $L, $S, $L)", Preconditions.class, paramName, ((Exactly) count).getCount(), errorMessage, paramName + ".length"));
+            }else if(count instanceof AtLeast){
+                c.addStatement(CodeBlock.of("$T.checkArgument($L.length >= $L, $S, $L)", Preconditions.class, paramName, ((AtLeast) count).getMin(), errorMessage, paramName + ".length"));
+            }else if(count instanceof AtMost){
+                c.addStatement(CodeBlock.of("$T.checkArgument($L.length <= $L, $S, $L)", Preconditions.class, paramName, ((AtMost) count).getMax(), errorMessage, paramName + ".length"));
+            }else if(count instanceof Range){
+                c.addStatement(CodeBlock.of("$T.checkArgument($L.length >= $L && $L.length <= $L, $S, $L)", Preconditions.class, paramName, ((Range) count).getFrom(), paramName, ((Range) count).getTo(), errorMessage, paramName + ".length"));
+            }
+        }
+    }
+
+    private static void generateEnums(File outputDirectory, String basePackage) throws IOException {
+        for (Arg it : Registry.INSTANCE.enums()) {
+            generateEnum(outputDirectory, "org.nd4j.enums", it);
+        }
+    }
+
+    private static String generateMethodText(Op op, Signature s, boolean isSameDiff, boolean isLoss, boolean withName) {
+        StringBuilder sb = new StringBuilder();
+        MethodSpec.Builder c = MethodSpec.methodBuilder(GenUtil.ensureFirstIsNotCap(op.getOpName()));
+        List<Parameter> params = s.getParameters();
+        List<Output> outs = op.getOutputs();
+        String retType = "void";
+
+        if (outs.size() == 1) {
+            retType = isSameDiff ? "SDVariable" : "INDArray";
+        }
+        else if (outs.size() >= 1) {
+            retType = isSameDiff ? "SDVariable[]" : "INDArray[]";
+        }
+        sb.append(retType + " " + op.getOpName() + "(");
+        boolean first = true;
+        for (Parameter param : params) {
+            if (param instanceof Arg) {
+                Arg arg = (Arg) param;
+                if (!first)
+                    sb.append(",");
+                else if (withName)
+                    sb.append("String name,");
+                TypeName tu = getArgType(arg);
+                sb.append(tu.toString() + " " + arg.name());
+                first = false;
+            }
+            else if (param instanceof Input) {
+                Input arg = (Input) param;
+                if (!first)
+                    sb.append(",");
+                else if (withName)
+                    sb.append("String name,");
+                sb.append((isSameDiff ? "SDVariable " : "INDArray ") + arg.name());
+                first = false;
+            }
+        }
+        sb.append(")");
+        return sb.toString();
+    }
+
+    private static StringBuilder buildDocSectionText(List<DocSection> docSections) {
+        StringBuilder sb = new StringBuilder();
+        for (DocSection ds : docSections) {
+            //if(ds.applies(Language.JAVA, CodeComponent.OP_CREATOR)){
+            String text = ds.getText();
+            String[] lines = text.split("\n");
+            for (int i = 0; i < lines.length; i++) {
+                if (!lines[i].endsWith("<br>")) {
+                    lines[i] = lines[i] + System.lineSeparator();
+                }
+            }
+            text = String.join("\n", lines);
+            sb.append(text + System.lineSeparator());
+            //}
+        }
+        return sb;
+    }
+
+    private static void generateDocs(NamespaceOps namespace, File outputDirectory, String basePackage) throws IOException {
+        StringBuilder sb = new StringBuilder();
+        sb.append("#  Namespace " + namespace.getName() + System.lineSeparator());
+        List<Op> ops = namespace.getOps();
+        for (Op op : ops) {
+            sb.append("## <a name=" + "\"").append(op.name()).append("\">").append(op.name()).append("</a>").append(System.lineSeparator());
+            List<DocSection> doc = op.getDoc();
+            if(!doc.isEmpty()) {
+                boolean first = true;
+                for(Signature s : op.getSignatures()) {
+                    if (first) {
+                        sb.append("````" + doc.get(0).getLanguage() + System.lineSeparator());
+                        first = false;
+                    }
+                    String ndCode = generateMethodText(op, s, false, false, false);
+                    sb.append(ndCode).append(System.lineSeparator());
+                    String sdCode = generateMethodText(op, s, true, false, false);
+                    sb.append(sdCode).append(System.lineSeparator());
+                    String withNameCode = generateMethodText(op, s, true, false, true);
+                    sb.append(withNameCode).append(System.lineSeparator());
+                }
+                sb.append("````").append(System.lineSeparator());
+                StringBuilder tsb = buildDocSectionText(doc);
+                sb.append(tsb.toString());
+                List<Signature> l = op.getSignatures();
+                for(Signature s : l) {
+                    List<Parameter> params = s.getParameters();
+                    for (Parameter p : params) {
+                        if(p instanceof Input){
+                            Input i = (Input)p;
+                            sb.append("* " + i.getName() + " " + (i.getDescription() == null ? "" : DocTokens.processDocText(i.getDescription(),
+                                    op, DocTokens.GenerationType.ND4J)) + " (" + i.getType() + " type)" + System.lineSeparator());
+                        } else if(p instanceof Arg) {
+                            Arg arg = (Arg) p;
+                            final Count count = arg.getCount();
+                            if (count == null || count.equals(exactlyOne)) {
+                                sb.append("* " + arg.getName() + " " + (arg.getDescription() == null ? "" : DocTokens.processDocText(arg.getDescription(),
+                                        op, DocTokens.GenerationType.ND4J)) +  System.lineSeparator());
+                            } else {
+                                sb.append("* " + arg.getName() + " " + (arg.getDescription() == null ? "" : DocTokens.processDocText(arg.getDescription(),
+                                        op, DocTokens.GenerationType.ND4J)) + " (Size: " + count.toString() +  System.lineSeparator());
+                            }
+                        }
+                    }
+                }
+                sb.append(System.lineSeparator());
+                tsb = buildDocSectionText(doc);
+                sb.append(tsb.toString());
+            }
+        }
+
+        for (Config config : Registry.INSTANCE.configs()) {
+            sb.append("## " + config.getName()  + System.lineSeparator());
+            boolean first = true;
+            for (Input i : config.getInputs()) {
+                if (first) {
+                    sb.append("````" + System.lineSeparator());
+                    first = false;
+                }
+                sb.append("* " + i.getName() + " " + i.getDescription() + " (" + i.getType() + " type)" + System.lineSeparator());
+            }
+            for (Arg arg : config.getArgs()) {
+                if (first) {
+                    sb.append("````" + System.lineSeparator());
+                    first = false;
+                }
+                sb.append("* " + arg.getName() + " " + " (" + arg.getType() + " type)" + System.lineSeparator());
+            }
+            StringBuilder tsb = buildDocSectionText(config.getDoc());
+            sb.append(tsb.toString());
+            sb.append("````" + System.lineSeparator());
+            ops.stream().filter(op -> op.getConfigs().contains(config)).forEach(op ->
+                    sb.append("[" + op.getOpName() + "]" + "(#" + op.getOpName() + ")" + System.lineSeparator()));
+        }
+        File outFile = new File(outputDirectory + "/ops", "/namespace-" + namespace.getName() + ".md");
+        FileUtils.writeStringToFile(outFile, sb.toString(), StandardCharsets.UTF_8);
+    }
+
+    private static void generateEnum(File outputDirectory, String targetPackage, Arg arg) throws IOException {
+        final String className = GenUtil.ensureFirstIsCap(arg.name());
+        enumMapping.put(arg, ClassName.get(targetPackage, className));
+
+        TypeSpec.Builder builder = TypeSpec.enumBuilder(className)
+                .addModifiers(Modifier.PUBLIC)
+                .addJavadoc(CodeBlock.of(arg.getDescription()));
+
+        for (String possibleValue : arg.getPossibleValues()) {
+            builder.addEnumConstant(possibleValue);
+        }
+
+        TypeSpec ts = builder.build();
+
+        JavaFile jf = JavaFile.builder(targetPackage, ts)
+                .build();
+
+
+        StringBuilder sb = new StringBuilder();
+        sb.append(copyright);
+        sb.append(codeGenWarning);
+        jf.writeTo(sb);
+
+        File outFile = new File(outputDirectory, packageToDirectory(targetPackage) + "/" + className + ".java");
+        FileUtils.writeStringToFile(outFile, sb.toString(), StandardCharsets.UTF_8);
+    }
+
+    private static void generateConfigs(File outputDirectory, String basePackage) throws IOException {
+        for (Config config : Registry.INSTANCE.configs()) {
+            generateConfig(outputDirectory, basePackage+".configs", config);
+        }
+    }
+
+    private static void generateConfig(File outputDirectory, String targetPackage, Config config) throws IOException {
+        if(config.getJavaClassOverride() != null && !config.getJavaClassOverride().isEmpty()){
+            //Java class override means "don't generate, use the existing one instead"
+            String c = config.getJavaClassOverride();
+            int idx = c.lastIndexOf('.');
+            String pkg = c.substring(0,idx);
+            String className = c.substring(idx+1);
+            configMapping.put(config, ClassName.get(pkg, className));
+            return;
+        }
+
+        final String className = GenUtil.ensureFirstIsCap(config.name());
+        configMapping.put(config, ClassName.get(targetPackage, className));
+
+        // Build Config Builder Class
+        final TypeSpec.Builder sdb = TypeSpec.classBuilder("SdBuilder").addModifiers(Modifier.STATIC, Modifier.PUBLIC);
+        final TypeSpec.Builder ndb = TypeSpec.classBuilder("NdBuilder").addModifiers(Modifier.STATIC, Modifier.PUBLIC);
+
+        for (Input input : config.getInputs()) {
+            addConfigBuilderParam(className, sdb, input.getName(), input.getType(), getType(TypeName.get(SDVariable.class), input.getCount()), input.getDescription(), input.getCount());
+            addConfigBuilderParam(className, ndb, input.getName(), input.getType(), getType(TypeName.get(INDArray.class), input.getCount()), input.getDescription(), input.getCount());
+        }
+
+        for (Arg arg : config.getArgs()) {
+            addConfigBuilderParam(className, sdb, arg.getName(), null, getArgType(arg), arg.getDescription(), arg.getCount());
+            addConfigBuilderParam(className, ndb, arg.getName(), null, getArgType(arg), arg.getDescription(), arg.getCount());
+        }
+
+        ArrayList<String> parts = new ArrayList<>();
+        ArrayList<Object> parameters = new ArrayList<>();
+        for (Input input : config.getInputs()) {
+            parts.add("$L");
+            parameters.add(
+                    input.hasDefaultValue() ?
+                            input.name() + " == null ? " + ((Input)input.defaultValue()).getName() +" : "+input.name()
+                            : input.name()
+            );        }
+        for (Arg input : config.getArgs()) {
+            parts.add("$L");
+            parameters.add(
+                    input.hasDefaultValue() ?
+                            input.name() + " == null ? " + anyToCode(input, input.defaultValue()) +" : "+input.name()
+                            : input.name()
+            );
+        }
+        parameters.add(0, className);
+
+        final MethodSpec.Builder build = MethodSpec.methodBuilder("build")
+                .addModifiers(Modifier.PUBLIC)
+                .returns(ClassName.bestGuess(className));
+        buildConstraints(build, config.getConstraints());
+        build.addStatement("return new $N("+(String.join(", ", parts))+")", parameters.toArray());
+
+        sdb.addMethod(build.build());
+        ndb.addMethod(build.build());
+
+
+        final TypeSpec ndBuilder = ndb.build();
+        final TypeSpec sdBuilder = sdb.build();
+
+
+        // Build Config Holder Class
+        TypeSpec.Builder holder = TypeSpec.classBuilder(className).addModifiers(Modifier.PUBLIC);
+
+        final MethodSpec.Builder ndConstructorBuilder = MethodSpec.constructorBuilder().addModifiers(Modifier.PRIVATE);
+        final MethodSpec.Builder sdConstructorBuilder = MethodSpec.constructorBuilder().addModifiers(Modifier.PRIVATE);
+
+
+        for (Input input : config.getInputs()) {
+            final String inputName = GenUtil.ensureFirstIsCap(input.getName());
+            addConfigParam(holder, ndConstructorBuilder, "nd" + inputName, getType(TypeName.get(INDArray.class), input.getCount()), input.getDescription(), true);
+            addConfigParam(holder, sdConstructorBuilder, "sd" + inputName, getType(TypeName.get(SDVariable.class), input.getCount()), input.getDescription(), true);
+        }
+
+        for (Arg arg : config.getArgs()) {
+            addConfigParam(holder, ndConstructorBuilder, arg.getName(), getArgType(arg), arg.getDescription(), true);
+            addConfigParam(holder, sdConstructorBuilder, arg.getName(), getArgType(arg), arg.getDescription(), false);
+        }
+        holder.addMethod(sdConstructorBuilder.build());
+        holder.addMethod(ndConstructorBuilder.build());
+
+        holder.addMethod(MethodSpec.methodBuilder("sdBuilder")
+                .addModifiers(Modifier.STATIC, Modifier.PUBLIC)
+                .addStatement("return new $N()", sdBuilder.name)
+                .returns(ClassName.bestGuess(sdBuilder.name))
+                .build());
+        holder.addType(sdBuilder);
+        holder.addMethod(MethodSpec.methodBuilder("ndBuilder")
+                .addModifiers(Modifier.STATIC, Modifier.PUBLIC)
+                .addStatement("return new $N()", ndBuilder.name)
+                .returns(ClassName.bestGuess(ndBuilder.name))
+                .build());
+        holder.addType(ndBuilder);
+
+        // add javadoc
+        //Method javadoc:
+        List<DocSection> doc = config.getDoc();
+        if(!doc.isEmpty()){
+            for(DocSection ds : doc){
+                if(ds.applies(Language.JAVA, CodeComponent.OP_CREATOR)){
+                    String text = ds.getText();
+                    //Add <br> tags at the end of each line, where none already exists
+                    String[] lines = text.split("\n");
+                    for( int i=0; i<lines.length; i++ ){
+                        if(!lines[i].endsWith("<br>")){
+                            lines[i] = lines[i] + "<br>";
+                        }
+                    }
+                    text = String.join("\n", lines);
+                    holder.addJavadoc(text + "\n\n");
+                }
+            }
+        }
+
+
+        // Document Constraints:
+        final List<Constraint> constraints = config.getConstraints();
+        if(!constraints.isEmpty()){
+            holder.addJavadoc("Inputs must satisfy the following constraints: <br>\n");
+            for (Constraint constraint : constraints) {
+                holder.addJavadoc(constraint.getMessage() +": " + constraintCodeGenerator.generateExpression(constraint.getCheck()) + "<br>\n");
+            }
+
+            holder.addJavadoc("\n");
+        }
+
+        TypeSpec ts = holder.build();
+
+
+        JavaFile jf = JavaFile.builder(targetPackage, ts)
+                .build();
+
+
+        StringBuilder sb = new StringBuilder();
+        sb.append(copyright);
+        sb.append(codeGenWarning);
+        jf.writeTo(sb);
+
+        File outFile = new File(outputDirectory, packageToDirectory(targetPackage) + "/" + className + ".java");
+        FileUtils.writeStringToFile(outFile, sb.toString(), StandardCharsets.UTF_8);
+    }
+
+    private static void addConfigParam(TypeSpec.Builder builder, MethodSpec.Builder constructorBuilder, String paramName, TypeName paramType, String paramDescription, boolean addField) {
+        if(addField){
+            // Add param fields
+            builder.addField(paramType, paramName, Modifier.PRIVATE);
+
+            // Add param getters
+            builder.addMethod(generateGetter(paramType, paramName, paramDescription, false));
+        }
+
+        // Add param constructor parameters
+        constructorBuilder.addParameter(paramType, paramName, Modifier.FINAL);
+        constructorBuilder.addStatement("this.$L = $L", paramName, paramName);
+    }
+
+    private static void addConfigBuilderParam(String configClassName, TypeSpec.Builder builder, String paramName, DataType inputType, TypeName paramType, String paramDescription, Count count) {
+        final String builderName = builder.build().name;
+        // Add param fields
+        builder.addField(paramType.box(), paramName, Modifier.PRIVATE);
+
+        // Add param getters
+        builder.addMethod(generateGetter(paramType, paramName, paramDescription, true));
+
+        // Add param setter
+        final MethodSpec.Builder setter = MethodSpec.methodBuilder(paramName)
+                .addParameter(paramType, paramName)
+                .addModifiers(Modifier.PUBLIC);
+        checkParameterCount(setter, count, paramName);
+        if(inputType != null){
+            if(builderName.equals("SdBuilder")){
+                setter.addStatement("$T.$L($S, $S, $L)", SDValidation.class, validationMapping.get(inputType), "Config: " + configClassName, paramName, paramName);
+            }else if(builderName.equals("NdBuilder")){
+                setter.addStatement("$T.$L($S, $S, $L)", NDValidation.class, validationMapping.get(inputType), "Config: " + configClassName, paramName, paramName);
+            }else{
+                throw new IllegalArgumentException("Unknown Builder Type "+builderName);
+            }
+        }
+        setter.addStatement("this.$L = $L", paramName, paramName)
+                .addStatement("return this")
+                .returns(ClassName.bestGuess(builderName));
+
+        if(count != null && !count.equals(exactlyOne)){
+            setter.varargs(true);
+        }
+
+        if(paramDescription != null){
+            setter.addJavadoc(paramDescription);
+        }
+        builder.addMethod(setter.build());
+    }
+
+    private static TypeName getType(TypeName typeVariable, Count count) {
+        if(count != null && !count.equals(exactlyOne)){
+            return ArrayTypeName.of(typeVariable);
+        }else{
+            return typeVariable;
+        }
+    }
+
+    @NotNull
+    private static MethodSpec generateGetter(TypeName typeVariable, String paramName, String paramDescription, boolean fluent) {
+        final MethodSpec.Builder getter = MethodSpec.methodBuilder((fluent ? paramName : "get" + GenUtil.ensureFirstIsCap(paramName)))
+                .addModifiers(Modifier.PUBLIC)
+                .returns(typeVariable);
+        if(paramDescription != null){
+            getter.addJavadoc(paramDescription);
+        }
+        getter.addStatement("return this.$L", paramName);
+        return getter.build();
+    }
+
+    private static String anyToCode(Parameter parameter, Object v){
+        if(v == null){ return "null"; }
+        else if(v instanceof int[]){ return "new int[]"+Arrays.toString((int[]) v).replace("[", "{").replace("]", "}"); }
+        else if(v instanceof long[]){ return "new long[]"+Arrays.toString((long[]) v).replace("[", "{").replace("]", "}"); }
+        else if(v instanceof float[]){ return "new float[]"+Arrays.toString((float[]) v).replace("[", "{").replace("]", "}"); }
+        else if(v instanceof double[]){ return "new double[]"+Arrays.toString((double[]) v).replace("[", "{").replace("]", "}"); }
+        else if(v instanceof boolean[]){ return "new boolean[]"+Arrays.toString((boolean[]) v).replace("[", "{").replace("]", "}"); }
+        else if(v instanceof Input){ return ((Input)v).getName(); }
+        else if(v instanceof org.nd4j.linalg.api.buffer.DataType){ return "DataType." + v; }
+        else if(v instanceof LossReduce || v instanceof org.nd4j.autodiff.loss.LossReduce){ return "org.nd4j.autodiff.loss.LossReduce." + v; }
+        else if(parameter instanceof Arg && ((Arg)parameter).getType() == DataType.ENUM){
+            return GenUtil.ensureFirstIsCap(parameter.name()) + "." + v.toString();
+        } else return v.toString();
+    }
+}

codegen/op-codegen/src/main/java/org/nd4j/codegen/impl/python/PythonGenerator.java

@@ -0,0 +1,164 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.impl.python;
+
+import org.apache.commons.io.FileUtils;
+import org.nd4j.codegen.api.*;
+import org.nd4j.codegen.api.doc.DocTokens;
+import org.nd4j.codegen.api.generator.Generator;
+import org.nd4j.codegen.api.generator.GeneratorConfig;
+import org.nd4j.codegen.util.GenUtil;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * This is a very simple, manual namespace generator
+ * We could of course use a templating library such as Freemarker, which woudl work fine - but:
+ * (a) on the one hand, it's overkill/unnecessary
+ * (b) on the other hand, may provide less flexibility than a manual implementation
+ *
+ */
+public class PythonGenerator implements Generator {
+
+    private static final String I4 = "    ";
+
+    @Override
+    public Language language() {
+        return Language.PYTHON;
+    }
+
+    @Override
+    public void generateNamespaceNd4j(NamespaceOps namespace, GeneratorConfig config, File directory, String fileName) throws IOException {
+
+
+        StringBuilder sb = new StringBuilder();
+        sb.append("class ").append(GenUtil.ensureFirstIsCap(namespace.getName())).append(":\n\n");
+
+        List<Op> ops = new ArrayList<>();
+        for(Op o : namespace.getOps()){
+            if(o.isAbstract())
+                continue;
+            ops.add(o);
+        }
+
+        //TODO: handle includes
+
+        for(Op o : ops){
+            String s = generateMethod(o);
+            sb.append(GenUtil.addIndent(s, 4));
+            sb.append("\n\n");
+        }
+
+        File f = new File(directory, GenUtil.ensureFirstIsCap(namespace.getName()) + ".py");
+        String content = sb.toString();
+
+        FileUtils.writeStringToFile(f, content, StandardCharsets.UTF_8);
+    }
+
+    protected static String generateMethod(Op op){
+        StringBuilder sb = new StringBuilder();
+        sb.append("@staticmethod\n")
+                .append("def ").append(GenUtil.ensureFirstIsNotCap(op.getOpName())).append("(");
+
+        //Add inputs to signature
+        boolean firstArg = true;
+        if(op.getInputs() != null){
+            for(Input i : op.getInputs()){
+                if(!firstArg)
+                    sb.append(", ");
+
+                sb.append(i.getName());
+
+                firstArg = false;
+            }
+        }
+
+
+        //Add arguments and default args to signature
+
+        sb.append("):\n");
+
+        String docString = genDocString(op);
+        sb.append(GenUtil.addIndent(docString, 4));
+
+        sb.append(I4).append("# Execution code here\n");
+
+
+        return sb.toString();
+    }
+
+    protected static String genDocString(Op op){
+        //Following roughly: https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html
+        StringBuilder sb = new StringBuilder();
+        sb.append("\"\"\"")
+                .append(op.getOpName())
+                .append(" operation")
+                .append("\n\n");
+        if(op.getInputs() != null){
+            sb.append("Args:");
+            sb.append("\n");
+            for(Input i : op.getInputs()){
+                sb.append(I4).append(i.getName()).append(" (ndarray): ");
+                if(i.getDescription() != null)
+                    sb.append(DocTokens.processDocText(i.getDescription(), op, DocTokens.GenerationType.ND4J));
+                sb.append("\n");
+            }
+        }
+
+        sb.append("\n");
+
+        if(op.getOutputs() != null){
+            sb.append("Returns:\n");
+            List<Output> o = op.getOutputs();
+
+            if(o.size() == 1){
+                sb.append(I4).append("ndarray: ").append(o.get(0).getName());
+                String d = o.get(0).getDescription();
+                if(d != null){
+                    sb.append(" - ").append(DocTokens.processDocText(d, op, DocTokens.GenerationType.ND4J));
+                }
+                sb.append("\n");
+            } else {
+                throw new UnsupportedOperationException("Not yet implemented: Python docstring generation for multiple output ops");
+            }
+        }
+
+        if(op.getArgs() != null){
+            //Args and default args
+            throw new UnsupportedOperationException("Generating method with args not yet implemented");
+        }
+
+        sb.append("\"\"\"\n");
+
+        return sb.toString();
+    }
+
+
+
+    @Override
+    public void generateNamespaceSameDiff(NamespaceOps namespace, GeneratorConfig config, File directory, String fileName) throws IOException {
+        throw new UnsupportedOperationException("Not yet implemented");
+    }
+}

codegen/op-codegen/src/main/java/org/nd4j/codegen/util/GenUtil.java

@@ -0,0 +1,64 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.util;
+
+import org.nd4j.codegen.api.Op;
+
+public class GenUtil {
+
+    private GenUtil(){ }
+
+    public static String ensureFirstIsCap(String in){
+        if(Character.isUpperCase(in.charAt(0))){
+           return in;
+        }
+
+        return Character.toUpperCase(in.charAt(0)) + in.substring(1);
+    }
+
+    public static String ensureFirstIsNotCap(String in){
+        if(Character.isLowerCase(in.charAt(0))){
+            return in;
+        }
+
+        return Character.toLowerCase(in.charAt(0)) + in.substring(1);
+    }
+
+    public static String repeat(String in, int count){
+        StringBuilder sb = new StringBuilder();
+        for( int i=0; i<count; i++ ){
+            sb.append(in);
+        }
+        return sb.toString();
+    }
+
+    public static String addIndent(String in, int count){
+        if(in == null)
+            return null;
+        String[] lines = in.split("\n");
+        StringBuilder out = new StringBuilder();
+        String indent = repeat(" ", count);
+        for(String s : lines){
+            out.append(indent).append(s).append("\n");
+        }
+        return out.toString();
+    }
+}

codegen/op-codegen/src/main/java/org/nd4j/codegen/util/JsonMapper.java

@@ -0,0 +1,46 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.util;
+
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+
+public class JsonMapper {
+
+    public static ObjectMapper getMapper(){
+        ObjectMapper om = new ObjectMapper();
+//        om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        om.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
+        om.enable(SerializationFeature.INDENT_OUTPUT);
+        om.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+        om.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
+        om.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
+        om.setVisibility(PropertyAccessor.CREATOR, JsonAutoDetect.Visibility.ANY);
+        om.setVisibility(PropertyAccessor.SETTER, JsonAutoDetect.Visibility.ANY);
+
+        return om;
+    }
+
+}

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/api/CodeComponent.kt

@@ -0,0 +1,23 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.api
+
+enum class CodeComponent { CLASS_DOC, CONSTRUCTOR, OP_CREATOR }

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/api/DataType.kt

@@ -0,0 +1,38 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.api
+
+enum class DataType {
+    NDARRAY,        // Any NDArray type (input only) - INDArray or SDVariable
+    FLOATING_POINT, // Any floating point data type
+    INT, // integer data type
+    LONG, //long, signed int64 datatype
+    NUMERIC, // any floating point or integer data type
+    BOOL, // boolean data type
+    STRING, //String value
+    // Arg only
+    DATA_TYPE, // tensor data type
+    CONDITION, // A condition
+    LOSS_REDUCE, // Loss reduction mode
+    ENUM; // defines an enum along with possibleValues property in Arg
+
+    fun isTensorDataType() = setOf(NDARRAY, FLOATING_POINT, INT, LONG, NUMERIC, BOOL).contains(this)
+}

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/api/Language.kt

@@ -0,0 +1,23 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.api
+
+enum class Language { ANY, JAVA, SCALA, KOTLIN, PYTHON, CPP }

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/api/LossReduce.kt

@@ -0,0 +1,31 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.api
+
+/**
+ *  See org.nd4j.autodiff.los.LossReduce in nd4j for documentation.
+ */
+enum class LossReduce {
+    NONE,
+    SUM,
+    MEAN_BY_WEIGHT,
+    MEAN_BY_NONZERO_WEIGHT_COUNT
+}

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/api/NamespaceOps.kt

@@ -0,0 +1,55 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.api
+
+data class NamespaceOps @JvmOverloads constructor(
+    var name: String,
+    var include: MutableList<String>? = null,
+    var ops: MutableList<Op> = mutableListOf(),
+    var configs: MutableList<Config> = mutableListOf(),
+    var parentNamespaceOps: Map<String,MutableList<Op>> = mutableMapOf()
+) {
+    fun addConfig(config: Config) {
+        configs.add(config)
+    }
+
+    /**
+     * Check that all required properties are set
+     */
+    fun checkInvariants() {
+        val usedConfigs = mutableSetOf<Config>()
+        ops.forEach { op ->
+            usedConfigs.addAll(op.configs)
+        }
+        val unusedConfigs = configs.toSet() - usedConfigs
+        if(unusedConfigs.size > 0){
+            throw IllegalStateException("Found unused configs: ${unusedConfigs.joinToString(", ") { it.name }}")
+        }
+    }
+
+    /**
+     * Get op by name
+     */
+    fun op(name:String):Op {
+        val op = ops.find { op -> op.opName.equals(name) } ?: throw java.lang.IllegalStateException("Operation $name not found")
+        return op
+    }
+}

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/api/Op.kt

@@ -0,0 +1,200 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.api
+
+import org.nd4j.codegen.api.doc.DocSection
+
+interface OpLike {
+    fun name(): String
+    fun inputs(): List<Input>
+    fun args(): List<Arg>
+    fun configs(): List<Config>
+    fun outputs(): List<Output>
+
+    fun allParameters(): List<Parameter>
+
+    fun addInput(input: Input)
+    fun addArgument(arg: Arg)
+    fun addOutput(output: Output)
+    fun addDoc(docs: DocSection)
+    fun addSignature(signature: Signature)
+    fun addConstraint(constraint: Constraint)
+    fun addConfig(config: Config)
+
+    fun Config.input(name: String) = inputs.find { it.name == name }!!
+    fun Config.arg(name: String) = args.find { it.name == name }!!
+
+    fun Mixin.input(name: String) = inputs.find { it.name == name }!!
+    fun Mixin.arg(name: String) = args.find { it.name == name }!!
+    fun Mixin.output(name: String) = outputs.find { it.name == name }!!
+    fun Mixin.config(name: String) = configs.find { it.name == name }!!
+}
+
+data class Op (
+        val opName: String,
+        var libnd4jOpName: String? = null,
+        var javaOpClass: String? = null,
+        var isAbstract: Boolean = false,
+        var legacy: Boolean = false,
+        var argsFirst: Boolean = false,
+        var javaPackage: String? = null,
+        val inputs: MutableList<Input> = mutableListOf(),
+        val outputs: MutableList<Output> = mutableListOf(),
+        val args: MutableList<Arg> = mutableListOf(),
+        val constraints: MutableList<Constraint> = mutableListOf(),
+        val signatures: MutableList<Signature> = mutableListOf(),
+        val doc: MutableList<DocSection> = mutableListOf(),
+        val configs: MutableList<Config> = mutableListOf()
+): OpLike {
+    override fun name() = opName
+    override fun inputs(): List<Input> = inputs
+    override fun args(): List<Arg> = args
+    override fun configs(): List<Config> = configs
+    override fun outputs(): List<Output> = outputs
+    override fun allParameters(): List<Parameter> = inputs + args + configs
+
+
+    override fun toString(): String {
+        return "Op(opName=$opName, libnd4jOpName=$libnd4jOpName, isAbstract=$isAbstract)"
+    }
+
+    override fun addInput(input: Input) { inputs.addOrReplace(input) }
+    override fun addArgument(arg: Arg) { args.addOrReplace(arg) }
+    override fun addOutput(output: Output) { outputs.addOrReplace(output) }
+    override fun addDoc(docs: DocSection){ doc.add(docs) }
+    override fun addSignature(signature: Signature){ signatures.add(signature) }
+    override fun addConstraint(constraint: Constraint){ constraints.add(constraint) }
+    override fun addConfig(config: Config) { configs.addOrReplace(config) }
+
+    /**
+     * Check that all required properties are set
+     */
+    fun checkInvariants() {
+        if( !isAbstract && (doc.size == 0 || doc.all { it.text.isNullOrBlank() } != false )){
+            throw IllegalStateException("$opName: Ops must be documented!")
+        }
+
+        signatures.forEach {
+            val opParameters = mutableListOf<Parameter>()
+            opParameters.addAll(inputs)
+            opParameters.addAll(args)
+
+            val notCovered = opParameters.fold(mutableListOf<Parameter>()){acc, parameter ->
+                if(!(it.parameters.contains(parameter) || parameter.defaultValueIsApplicable(it.parameters))){
+                    acc.add(parameter)
+                }
+                acc
+            }
+
+            if(notCovered.size > 0){
+                throw IllegalStateException("$opName: $it does not cover all parameters! Missing: ${notCovered.joinToString(", ") { it.name() }}")
+            }
+        }
+
+        args.filter { it.type == DataType.ENUM }.forEach {
+            if(it.description == null){
+                throw IllegalStateException("$opName: Argument ${it.name} is ENUM but has no documentation!")
+            }
+        }
+    }
+}
+
+data class Mixin (
+        val name: String,
+
+        val inputs: MutableList<Input> = mutableListOf(),
+        val outputs: MutableList<Output> = mutableListOf(),
+        val args: MutableList<Arg> = mutableListOf(),
+        val constraints: MutableList<Constraint> = mutableListOf(),
+        val signatures: MutableList<Signature> = mutableListOf(),
+        val doc: MutableList<DocSection> = mutableListOf(),
+        val configs: MutableList<Config> = mutableListOf()
+): OpLike {
+    override fun name() = name
+    override fun inputs(): List<Input> = inputs
+    override fun args(): List<Arg> = args
+    override fun configs(): List<Config> = configs
+    override fun outputs(): List<Output> = outputs
+    override fun allParameters(): List<Parameter> = inputs + args + configs
+
+    override fun toString(): String {
+        return "Mixin($name)"
+    }
+
+    override fun addInput(input: Input) { inputs.addOrReplace(input) }
+    override fun addArgument(arg: Arg) { args.addOrReplace(arg) }
+    override fun addOutput(output: Output) { outputs.addOrReplace(output) }
+    override fun addDoc(docs: DocSection){ doc.add(docs) }
+    override fun addSignature(signature: Signature){ signatures.add(signature) }
+    override fun addConstraint(constraint: Constraint){ constraints.add(constraint) }
+    override fun addConfig(config: Config) { configs.addOrReplace(config) }
+
+
+    var legacyWasSet: Boolean = false
+    var legacy: Boolean = false
+        set(value) {
+            field = value
+            legacyWasSet = true
+        }
+    var javaPackageWasSet: Boolean = false
+    var javaPackage: String? = null
+        set(value) {
+            field = value
+            javaPackageWasSet = true
+        }
+
+    /**
+     * Check that all required properties are set
+     */
+    fun checkInvariants() {
+        signatures.forEach {
+            val opParameters = mutableListOf<Parameter>()
+            opParameters.addAll(inputs)
+            opParameters.addAll(args)
+
+            val notCovered = opParameters.fold(mutableListOf<Parameter>()){acc, parameter ->
+                if(!(it.parameters.contains(parameter) || parameter.defaultValueIsApplicable(it.parameters))){
+                    acc.add(parameter)
+                }
+                acc
+            }
+
+            if(notCovered.size > 0){
+                throw IllegalStateException("$this: $it does not cover all parameters! Missing: ${notCovered.joinToString(", ") { it.name() }}")
+            }
+        }
+    }
+}
+
+fun <T: Parameter> MutableList<T>.addOrReplaceAll(params: List<T>){
+    params.forEach {
+        this.addOrReplace(it)
+    }
+}
+
+fun <T: Parameter> MutableList<T>.addOrReplace(param: T){
+    val found = this.find { it.name() == param.name() }
+    if(found != null){
+        this.replaceAll { if(it.name() == param.name()){ param } else { it } }
+    }else{
+        this.add(param)
+    }
+}

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/api/Registry.kt

@@ -0,0 +1,45 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.api
+
+object Registry {
+    private val enums: MutableMap<String, Arg> = mutableMapOf()
+    private val configs: MutableMap<String, Config> = mutableMapOf()
+
+    fun enums() = enums.values.sortedBy { it.name }
+    fun configs() = configs.values.sortedBy { it.name }
+
+    fun registerEnum(arg: Arg){
+        when(enums[arg.name]){
+            null -> enums[arg.name] = arg
+            arg -> { /* noop */ }
+            else -> throw IllegalStateException("Another enum with the name ${arg.name} already exists! Enums have to use unique names. If you want to use an enum in multiple places, use mixins to define them.")
+        }
+    }
+
+    fun registerConfig(config: Config){
+        when(configs[config.name]){
+            null -> configs[config.name] = config
+            config -> { /* noop */ }
+            else -> throw IllegalStateException("Another config with the name ${config.name} already exists! Configs have to use unique names.")
+        }
+    }
+}

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/api/Variables.kt

@@ -0,0 +1,260 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.api
+
+import org.nd4j.codegen.api.doc.DocSection
+import java.util.*
+
+
+open class Constraint (
+        var message: String? = null,
+        var check: Expression
+)
+
+class BackendConstraint(message: String? = null, check: Expression): Constraint(message, check)
+
+// Used in Constraint Expressions
+sealed class Reference
+data class NumberReference<T: Number>(val value: T): Reference()
+data class BooleanReference(val value: Boolean): Reference()
+data class InputShapeReference(val input: Input, val idx: Int): Reference()
+data class InputRankReference(val input: Input): Reference()
+sealed class Expression: Reference()
+data class BooleanExpression(val left: Reference, val right: Reference, val op: BooleanOperation): Expression()
+data class SameTypeExpression(val inputs: List<Input>): Expression()
+data class SameShapeExpression(val inputs: List<Input>): Expression()
+data class BroadcastableShapesExpression(val inputs: List<Input>): Expression()
+enum class BooleanOperation{EQ, NEQ, LT, LTE, GT, GTE, AND, OR}
+
+
+// Used to define array sizes
+sealed class Count
+data class Range(val from: Int, val to: Int): Count()
+data class AtLeast(val min: Int): Count()
+data class AtMost(val max: Int): Count()
+data class Exactly(val count: Int): Count()
+
+// Actual parameters
+interface Parameter {
+    fun name(): String
+    fun defaultValue() : Any?
+
+    fun hasDefaultValue(): Boolean
+
+    fun isVararg():Boolean
+
+    /**
+     * A default value only is applicable if it is a literal value, or the referenced value is either directly a part of
+     * the signature, or there is a reference chain that ends in something that is actually a part of the signature
+     */
+    fun defaultValueIsApplicable(otherParams: List<Parameter>): Boolean = if(hasDefaultValue()){
+        when(val defaultValue = this.defaultValue()){
+            is Number, is Boolean, null -> true
+            is IntArray, is BooleanArray, is DoubleArray -> true
+            is String -> true
+            is org.nd4j.linalg.api.buffer.DataType -> true
+            is org.nd4j.codegen.api.LossReduce -> true
+            is Parameter -> otherParams.contains(defaultValue) || defaultValue.defaultValueIsApplicable(otherParams)
+            is TensorDataTypeValue -> otherParams.contains(defaultValue.tensor) || defaultValue.tensor.defaultValueIsApplicable(otherParams)
+            is TensorShapeValue -> otherParams.contains(defaultValue.tensor) || defaultValue.tensor.defaultValueIsApplicable(otherParams)
+            else -> false
+        }
+    }else{
+        false
+    }
+}
+interface Tensor: Parameter
+
+data class Arg(
+        val name: String,
+        val type: DataType,
+        var description: String? = null,
+        var isVargarg: Boolean = false
+) : Reference(), Parameter {
+    override fun name(): String = name
+    override fun defaultValue(): Any? = defaultValue
+    override fun hasDefaultValue(): Boolean = defaultValueIsSet
+    override fun isVararg(): Boolean {
+        return isVargarg
+    }
+
+    private var defaultValueIsSet = false
+    var defaultValue: Any? = null
+        set(value) = if(isAssignableFrom(value)) {
+            field = value
+            defaultValueIsSet = true
+        }else{
+            throw IllegalArgumentException("Illegal default value for $this. Got ${value.toDescriptiveString()} (${value?.javaClass?.name})")
+        }
+
+    var possibleValues: List<String>? = null
+        set(value) = if(type == DataType.ENUM) when {
+            value == null -> field = null
+            value.isEmpty() -> throw IllegalArgumentException("$this: Can not set empty possibleValues.")
+            else -> field = value
+        } else {
+            throw IllegalArgumentException("$this: Can not set possibleValues on non ENUM typed Arg.")
+        }
+
+    var count: Count? = null
+        set(value) = if(type == DataType.ENUM && value != Exactly(1)) {
+            throw IllegalArgumentException("$this: ENUM typed Arg can not be array")
+        }else{
+            field = value
+        }
+
+    private fun matchesDataType(value: Any?) = when(type){
+        DataType.FLOATING_POINT -> value is Double
+        DataType.INT -> (value is Int) || (value is Long)
+        DataType.LONG -> (value is Int) || (value is Long)
+        DataType.NUMERIC -> value is Number
+        DataType.BOOL -> value is Boolean
+        else -> false
+    }
+
+    private fun isAssignableFrom(value: Any?) = when(value){
+        is TensorShapeValue -> isArray() && type == DataType.INT
+        is TensorDataTypeValue -> type == DataType.DATA_TYPE
+        is Number, is Boolean -> matchesDataType(value)
+        is IntArray -> isArray() && (type == DataType.INT || type == DataType.NUMERIC) && countMatches(value.size)
+        is DoubleArray -> isArray() && (type == DataType.FLOATING_POINT || type == DataType.NUMERIC) && countMatches(value.size)
+        is BooleanArray -> isArray() && type == DataType.BOOL && countMatches(value.size)
+        is Arg -> value.count == count && value.type == type
+        is String -> type == DataType.STRING || type == DataType.ENUM && possibleValues != null && possibleValues?.contains(value) ?: false
+        //is String -> type == DataType.ENUM && possibleValues != null && possibleValues?.contains(value) ?: false
+        is org.nd4j.linalg.api.buffer.DataType -> type == DataType.DATA_TYPE
+        is org.nd4j.codegen.api.LossReduce -> type == DataType.LOSS_REDUCE
+        null -> true
+        else -> false
+    }
+
+    fun isArray() = count != Exactly(1) && count != null
+    fun countMatches(size: Int) = when(val c = count!!){
+        is Range -> c.from <= size && size <= c.to
+        is AtLeast -> c.min <= size
+        is AtMost -> size <= c.max
+        is Exactly -> c.count == size
+    }
+
+    fun Tensor.shape() = TensorShapeValue(this)
+    fun Tensor.dataType() = TensorDataTypeValue(this)
+
+    override fun toString() = "Arg(${if(type == DataType.ENUM){
+        "ENUM(${possibleValues?.joinToString(", ")})"
+    }else{
+        type.toString()
+    }}, $name)${if(count != null) "{ count = $count }" else "" }"
+}
+
+data class Input (
+        val name: String,
+        val type: DataType,
+        var description: String? = null,
+        var count: Count? = null
+) : Parameter, Tensor {
+    override fun isVararg(): Boolean {
+        return false
+    }
+
+    override fun name(): String = name
+    override fun defaultValue(): Any? = defaultValue
+    override fun hasDefaultValue(): Boolean = defaultValueIsSet
+
+    private var defaultValueIsSet = false
+    var defaultValue: Input? = null
+        set(value) = if(matchesDataType(value)){
+            field = value
+            defaultValueIsSet = true
+        }else{
+            throw IllegalArgumentException("Illegal default value for Input($name). Allowed values have to match data type $type, but got ${value.toDescriptiveString()} (${value?.javaClass?.name})")
+        }
+
+    private fun matchesDataType(value: Input?) = when(value){
+        null -> true
+        else -> value.type == type
+    }
+}
+
+data class Output(
+        var name: String,
+        var type: DataType,
+        var multiOutput: Boolean,
+        var description: String? = null
+) : Parameter, Tensor{
+    override fun isVararg(): Boolean {
+        return false
+    }
+
+    override fun name(): String = name
+    override fun defaultValue(): Any? = null
+    override fun hasDefaultValue(): Boolean = false
+}
+
+data class Signature(
+        val parameters: List<Parameter>,
+        val description: String? = null
+){
+    override fun toString(): String {
+        return "Signature(${parameters.joinToString {it.name()}})"
+    }
+}
+
+// Used in defining default values
+data class TensorShapeValue(val tensor: Tensor) {
+    override fun toString(): String = "${tensor.name()}.shape()"
+}
+data class TensorDataTypeValue(val tensor: Tensor){
+    override fun toString(): String = "${tensor.name()}.dataType()"
+}
+
+fun Any?.toDescriptiveString() = when(this){
+    null -> "null"
+    is IntArray -> Arrays.toString(this)
+    is LongArray -> Arrays.toString(this)
+    is DoubleArray -> Arrays.toString(this)
+    is FloatArray -> Arrays.toString(this)
+    is BooleanArray -> Arrays.toString(this)
+    is Array<*> -> Arrays.toString(this)
+    else -> this.toString()
+}
+
+data class Config(
+        val name: String,
+        val inputs: MutableList<Input> = mutableListOf(),
+        val args: MutableList<Arg> = mutableListOf(),
+        val constraints: MutableList<Constraint> = mutableListOf(),
+        val doc: MutableList<DocSection> = mutableListOf()
+        ): Parameter {
+    override fun isVararg(): Boolean {
+        return false
+    }
+
+    override fun name(): String = name
+    override fun defaultValue(): Any? = null
+    override fun hasDefaultValue(): Boolean = false
+
+    fun addInput(input: Input) { inputs.add(input) }
+    fun addArgument(arg: Arg) { args.add(arg) }
+    fun addConstraint(constraint: Constraint){ constraints.add(constraint) }
+    fun addDoc(doc: DocSection){ this.doc.add(doc) }
+
+    var javaClassOverride: String = ""
+}

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/api/doc/DocScope.kt

@@ -0,0 +1,36 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.api.doc
+
+import org.nd4j.codegen.api.CodeComponent
+
+enum class DocScope {
+    ALL, CLASS_DOC_ONLY, CREATORS_ONLY, CONSTRUCTORS_ONLY;
+
+    fun applies(codeComponent: CodeComponent): Boolean {
+        return when (this) {
+            ALL -> true
+            CLASS_DOC_ONLY -> codeComponent === CodeComponent.CLASS_DOC
+            CREATORS_ONLY -> codeComponent === CodeComponent.OP_CREATOR
+            CONSTRUCTORS_ONLY -> codeComponent === CodeComponent.CONSTRUCTOR
+        }
+    }
+}

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/api/doc/DocSection.kt

@@ -0,0 +1,34 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.api.doc
+
+import org.nd4j.codegen.api.CodeComponent
+import org.nd4j.codegen.api.Language
+
+
+data class DocSection(var scope: DocScope? = null,
+                 var language: Language? = null,
+                 var text: String? = null) {
+
+    fun applies(language: Language, codeComponent: CodeComponent): Boolean {
+        return (this.language === Language.ANY || language === this.language) && scope!!.applies(codeComponent)
+    }
+}

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/api/doc/DocTokens.kt

@@ -0,0 +1,40 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.api.doc
+
+import org.nd4j.codegen.api.Op
+
+object DocTokens {
+    enum class GenerationType { SAMEDIFF, ND4J }
+    private val OPNAME = "%OPNAME%".toRegex()
+    private val LIBND4J_OPNAME = "%LIBND4J_OPNAME%".toRegex()
+    private val INPUT_TYPE = "%INPUT_TYPE%".toRegex()
+
+    @JvmStatic fun processDocText(doc: String?, op: Op, type: GenerationType): String {
+        return doc
+                ?.replace(OPNAME, op.opName)
+                ?.replace(LIBND4J_OPNAME, op.libnd4jOpName!!)
+                ?.replace(INPUT_TYPE, when(type){
+                    GenerationType.SAMEDIFF -> "SDVariable"
+                    GenerationType.ND4J -> "INDArray"
+                }) ?: ""
+    }
+}

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/api/generator/ConstraintCodeGenerator.kt

@@ -0,0 +1,28 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.api.generator
+
+import org.nd4j.codegen.api.Expression
+
+
+interface ConstraintCodeGenerator {
+    fun generateExpression(expression: Expression): String
+}

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/api/generator/Generator.kt

@@ -0,0 +1,36 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.api.generator
+
+import org.nd4j.codegen.api.Language
+import org.nd4j.codegen.api.NamespaceOps
+import java.io.File
+import java.io.IOException
+
+interface Generator {
+    fun language(): Language?
+
+    @Throws(IOException::class)
+    fun generateNamespaceNd4j(namespace: NamespaceOps?, config: GeneratorConfig?, directory: File?, className: String?)
+
+    @Throws(IOException::class)
+    fun generateNamespaceSameDiff(namespace: NamespaceOps?, config: GeneratorConfig?, directory: File?, className: String?)
+}

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/api/generator/GeneratorConfig.kt

@@ -0,0 +1,27 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.api.generator
+
+import org.nd4j.codegen.api.Op
+
+class GeneratorConfig {
+    fun acceptOp(op: Op?): Boolean = true
+}

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/dsl/OpBuilder.kt

@@ -0,0 +1,370 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.dsl
+
+import org.nd4j.codegen.api.*
+import org.nd4j.codegen.api.doc.DocScope
+import org.nd4j.codegen.api.doc.DocSection
+import org.nd4j.codegen.ops.SDBaseOps
+import java.lang.IllegalStateException
+
+fun Namespace(name: String, block: NamespaceOps.() -> Unit): NamespaceOps {
+    val ns = NamespaceOps(name)
+    ns.block()
+
+    ns.checkInvariants()
+    return ns
+}
+
+fun Mixin(name: String, block: Mixin.() -> Unit): Mixin {
+    return Mixin(name).apply(block).also {
+        it.checkInvariants()
+    }
+}
+
+fun NamespaceOps.Alias(ns:NamespaceOps, opName:String):Op {
+        val op:Op? = ns.op(opName)
+        op?.let {
+            this.parentNamespaceOps[ns.name]?.add(op)
+            this.ops.add(op)
+            return op
+        }
+        throw IllegalStateException("Failed to create alias for op: $opName")
+}
+
+fun NamespaceOps.Op(name: String, block: Op.() -> Unit): Op {
+    val op = Op(name)
+    op.libnd4jOpName = name
+
+    op.block()
+
+    if (!op.isAbstract && op.signatures.isEmpty()) {
+        op.AllParamSignature()
+        op.AllDefaultsSignature()
+    }
+
+    op.checkInvariants()
+
+    this.ops.add(op)
+    return op
+}
+
+fun NamespaceOps.Op(name: String,
+                    extends: Mixin,
+                    keepArgs: Boolean = true,
+                    keepInputs: Boolean = true,
+                    keepOutputs: Boolean = true,
+                    keepConstraints: Boolean = true,
+                    keepSignatures: Boolean = true,
+                    keepDocs: Boolean = true,
+                    block: (Op.() -> Unit)? = null): Op {
+    return this.Op(name) {
+        useMixin(extends, keepArgs = keepArgs, keepInputs = keepInputs, keepOutputs = keepOutputs, keepConstraints = keepConstraints, keepSignatures = keepSignatures, keepDocs = keepDocs)
+
+        if (block != null) {
+            this.block()
+        }
+    }
+}
+
+
+fun OpLike.Input(dataType: DataType, name: String, block: (Input.() -> Unit)? = null): Input {
+    val input = Input(name, dataType)
+    if (block != null) input.block()
+
+    if (!dataType.isTensorDataType()) {
+        throw IllegalArgumentException("Invalid datatype for input \"$name\" of op ${this.name()}: inputs arrays cannot have type $dataType - wrong type, or should be Arg type?");
+    }
+
+    this.addInput(input)
+
+
+    return input
+}
+
+fun OpLike.Arg(dataType: DataType, name: String, block: (Arg.() -> Unit)? = null): Arg {
+    val input = Arg(name, dataType)
+    if (block != null) input.block()
+
+    this.addArgument(input)
+    if(dataType == DataType.ENUM){
+        Registry.registerEnum(input)
+    }
+    return input
+}
+
+fun OpLike.Output(dataType: DataType, name: String, block: (Output.() -> Unit)? = null): Output {
+    val output = Output(name, dataType, false)
+    if (block != null) output.block()
+
+    if (!dataType.isTensorDataType()) {
+        throw IllegalArgumentException("Invalid datatype for output \"$name\" of op ${this.name()}: output arrays cannot have type $dataType");
+    }
+
+    this.addOutput(output)
+    return output
+}
+
+fun OpLike.Doc(language: Language, scope: DocScope, block: DocSection.() -> String): DocSection {
+    val doc = DocSection().apply {
+        this.language = language
+        this.scope = scope
+        text = this.block()
+    }
+    this.addDoc(doc)
+    return doc
+}
+
+fun OpLike.AllParamSignature(withOutput: Boolean = false) {
+    val allParameters = allParameters()
+
+    this.addSignature(Signature(allParameters))
+    if (withOutput) {
+        val withOutputParams = mutableListOf<Parameter>().also {
+            it.addAll(this.outputs())
+            it.addAll(allParameters)
+        }
+        this.addSignature(Signature(withOutputParams))
+    }
+}
+
+fun OpLike.AllDefaultsSignature(withOutput: Boolean = false) {
+    val allParameters = allParameters()
+
+    if (allParameters.find { it.hasDefaultValue() } != null) {
+        val params = allParameters.filterNot { it.hasDefaultValue() }
+        this.addSignature(Signature(params))
+        if (withOutput) {
+            val withOutputParams = mutableListOf<Parameter>().also {
+                it.addAll(this.outputs())
+                it.addAll(allParameters)
+            }
+            this.addSignature(Signature(withOutputParams))
+        }
+    }
+}
+
+fun OpLike.Signature(vararg params: Parameter, block: (Signature.() -> String)? = null): Signature {
+    if (params.toSet().size < params.size) {
+        throw IllegalArgumentException("A parameter may not be used twice in a signature!")
+    }
+    val paramsAndOutputs = allParameters() + outputs()
+    if (!paramsAndOutputs.containsAll(params.toList())) {
+        throw IllegalArgumentException("You can only use parameters in a signature that are actually defined in $this! Did you forget to useMixin(...) a mixin?")
+    }
+
+    val signature = Signature(params.toList())
+    if (block != null) {
+        signature.block()
+    }
+    this.addSignature(signature)
+    return signature
+}
+
+fun OpLike.Constraint(desc: String, block: ConstraintBuilder.() -> Expression): Constraint {
+    val check = ConstraintBuilder().block()
+    val constraint = Constraint(desc, check)
+    this.addConstraint(constraint)
+    return constraint
+}
+
+fun OpLike.BackendConstraint(desc: String, block: ConstraintBuilder.() -> Expression): Constraint {
+    val check = ConstraintBuilder().block()
+    val constraint = BackendConstraint(desc, check)
+    this.addConstraint(constraint)
+    return constraint
+}
+
+class ConstraintBuilder {
+    fun broadcastableShapes(vararg inputs: Input) = BroadcastableShapesExpression(inputs.toList())
+    fun sameShape(vararg inputs: Input) = SameShapeExpression(inputs.toList())
+    fun sameType(vararg inputs: Input) = SameTypeExpression(inputs.toList())
+
+    fun Input.sizeAt(i: Int) = InputShapeReference(this, i)
+    fun Input.rank() = InputRankReference(this)
+    fun Input.isScalar() = this.rank() eq 0
+
+    fun some(expr: BooleanExpression, vararg exprs: BooleanExpression) = exprs.fold(expr, { acc, cur -> acc or cur })
+    fun all(expr: BooleanExpression, vararg exprs: BooleanExpression) = exprs.fold(expr, { acc, cur -> acc and cur })
+    fun not(expr: BooleanExpression) = expr eq false
+
+    infix fun BooleanExpression.and(other: BooleanExpression) = BooleanExpression(this, other, BooleanOperation.AND)
+    infix fun BooleanExpression.or(other: BooleanExpression) = BooleanExpression(this, other, BooleanOperation.OR)
+
+
+    infix fun Reference.eq(other: Reference) = BooleanExpression(this, other, BooleanOperation.EQ)
+    infix fun Reference.eq(other: Number) = this eq NumberReference(other)
+    infix fun Reference.eq(other: Boolean) = this eq BooleanReference(other)
+
+
+    infix fun Reference.neq(other: Reference) = BooleanExpression(this, other, BooleanOperation.NEQ)
+    infix fun <T : Number> Reference.neq(other: T) = this neq NumberReference(other)
+    infix fun Reference.neq(other: Boolean) = this neq BooleanReference(other)
+
+    infix fun Reference.gt(other: Reference) = BooleanExpression(this, other, BooleanOperation.GT)
+    infix fun <T : Number> Reference.gt(other: T) = this gt NumberReference(other)
+
+    infix fun Reference.lt(other: Reference) = BooleanExpression(this, other, BooleanOperation.LT)
+    infix fun <T : Number> Reference.lt(other: T) = this lt NumberReference(other)
+
+
+    infix fun <T : Number> Reference.gte(other: T) = this gte NumberReference(other)
+    infix fun Reference.gte(other: Reference) = BooleanExpression(this, other, BooleanOperation.GTE)
+
+    infix fun <T : Number> Reference.lte(other: T) = this lte NumberReference(other)
+    infix fun Reference.lte(other: Reference) = BooleanExpression(this, other, BooleanOperation.LTE)
+}
+
+fun NamespaceOps.Config(name: String, block: (Config.() -> Unit)): Config {
+    val config = Config(name)
+    config.block()
+    this.addConfig(config)
+    Registry.registerConfig(config)
+    return config
+}
+
+fun Config.Input(dataType: DataType, name: String, block: (Input.() -> Unit)? = null): Input {
+    val input = Input(name, dataType)
+    if (block != null) input.block()
+
+    if (!dataType.isTensorDataType()) {
+        throw IllegalArgumentException("Invalid datatype for input \"$name\" of config ${this.name}: inputs arrays cannot have type $dataType - wrong type, or should be Arg type?");
+    }
+
+    this.addInput(input)
+    return input
+}
+
+fun Config.Arg(dataType: DataType, name: String, block: (Arg.() -> Unit)? = null): Arg {
+    val input = Arg(name, dataType)
+    if (block != null) input.block()
+
+    this.addArgument(input)
+    if(dataType == DataType.ENUM){
+        Registry.registerEnum(input)
+    }
+
+    return input
+}
+
+fun Config.Constraint(desc: String, block: ConstraintBuilder.() -> Expression): Constraint {
+    val check = ConstraintBuilder().block()
+    val constraint = Constraint(desc, check)
+    this.addConstraint(constraint)
+    return constraint
+}
+
+fun Config.BackendConstraint(desc: String, block: ConstraintBuilder.() -> Expression): Constraint {
+    val check = ConstraintBuilder().block()
+    val constraint = BackendConstraint(desc, check)
+    this.addConstraint(constraint)
+    return constraint
+}
+
+fun Config.Doc(language: Language, scope: DocScope, block: DocSection.() -> String): DocSection {
+    val doc = DocSection().apply {
+        this.language = language
+        this.scope = scope
+        text = this.block()
+    }
+    this.addDoc(doc)
+    return doc
+}
+
+fun OpLike.useConfig(config: Config): Config {
+    this.addConfig(config)
+    return config
+}
+
+fun Op.useMixin(mixin: Mixin,
+                keepArgs: Boolean = true,
+                keepInputs: Boolean = true,
+                keepOutputs: Boolean = true,
+                keepConstraints: Boolean = true,
+                keepSignatures: Boolean = true,
+                keepDocs: Boolean = true,
+                keepConfigs: Boolean = true
+) {
+    if(mixin.legacyWasSet){
+        legacy = mixin.legacy
+    }
+    if(mixin.javaPackageWasSet){
+        javaPackage = mixin.javaPackage
+    }
+    if (keepArgs) {
+        args.addOrReplaceAll(mixin.args)
+    }
+    if (keepInputs) {
+        inputs.addOrReplaceAll(mixin.inputs)
+    }
+    if (keepOutputs) {
+        outputs.addOrReplaceAll(mixin.outputs)
+    }
+    if (keepConstraints) {
+        constraints.addAll(mixin.constraints)
+    }
+    if (keepSignatures) {
+        signatures.addAll(mixin.signatures)
+    }
+    if (keepDocs) {
+        doc.addAll(mixin.doc)
+    }
+    if(keepConfigs){
+        configs.addOrReplaceAll(mixin.configs)
+    }
+}
+
+fun Mixin.useMixin(mixin: Mixin,
+                   keepArgs: Boolean = true,
+                   keepInputs: Boolean = true,
+                   keepOutputs: Boolean = true,
+                   keepConstraints: Boolean = true,
+                   keepSignatures: Boolean = true,
+                   keepDocs: Boolean = true,
+                   keepConfigs: Boolean = true) {
+    if(mixin.legacyWasSet){
+        legacy = mixin.legacy
+    }
+    if(mixin.javaPackageWasSet){
+        javaPackage = mixin.javaPackage
+    }
+    if (keepArgs) {
+        args.addOrReplaceAll(mixin.args)
+    }
+    if (keepInputs) {
+        inputs.addOrReplaceAll(mixin.inputs)
+    }
+    if (keepOutputs) {
+        outputs.addOrReplaceAll(mixin.outputs)
+    }
+    if (keepConstraints) {
+        constraints.addAll(mixin.constraints)
+    }
+    if (keepSignatures) {
+        signatures.addAll(mixin.signatures)
+    }
+    if (keepDocs) {
+        doc.addAll(mixin.doc)
+    }
+    if(keepConfigs){
+        configs.addOrReplaceAll(mixin.configs)
+    }
+}

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/impl/java/JavaConstraintCodeGenerator.kt

@@ -0,0 +1,55 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.impl.java
+
+import org.nd4j.codegen.api.*
+import org.nd4j.codegen.api.generator.ConstraintCodeGenerator
+
+class JavaConstraintCodeGenerator: ConstraintCodeGenerator {
+    override fun generateExpression(expression: Expression): String = when(expression) {
+        is BooleanExpression -> {
+            val left = generateReference(expression.left)
+            val right = generateReference(expression.right)
+            when(expression.op){
+                BooleanOperation.EQ -> "$left == $right"
+                BooleanOperation.NEQ -> "$left != $right"
+                BooleanOperation.LT -> "$left < $right"
+                BooleanOperation.LTE -> "$left <= $right"
+                BooleanOperation.GT -> "$left > $right"
+                BooleanOperation.GTE -> "$left >= $right"
+                BooleanOperation.AND -> "$left && $right"
+                BooleanOperation.OR -> "$left || $right"
+            }
+        }
+        is SameTypeExpression -> "isSameType(${expression.inputs.joinToString(", "){ it.name }})"
+        is SameShapeExpression -> "isSameShape(${expression.inputs.joinToString(", "){ it.name }})"
+        is BroadcastableShapesExpression -> "isBroadcastableShapes(${expression.inputs.joinToString(", "){ it.name }})"
+    }
+
+    private fun generateReference(reference: Reference): String = when(reference){
+        is NumberReference<*> -> reference.value.toString()
+        is BooleanReference -> reference.value.toString()
+        is InputShapeReference -> "${reference.input.name}.sizeAt(${reference.idx})"
+        is InputRankReference -> "${reference.input.name}.rank()"
+        is Arg -> reference.name
+        is Expression -> "(${generateExpression(reference)})"
+    }
+}

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/impl/python/KotlinExamplePythonGenerator.kt

@@ -0,0 +1,94 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.impl.python
+
+import org.apache.commons.io.FileUtils
+import org.nd4j.codegen.api.Language
+import org.nd4j.codegen.api.NamespaceOps
+import org.nd4j.codegen.api.Op
+import org.nd4j.codegen.api.doc.DocTokens
+import org.nd4j.codegen.api.generator.Generator
+import org.nd4j.codegen.api.generator.GeneratorConfig
+import org.nd4j.codegen.util.GenUtil
+import java.io.File
+import java.io.IOException
+import java.nio.charset.StandardCharsets
+
+class KotlinExamplePythonGenerator: Generator {
+    override fun language() = Language.PYTHON
+
+    @Throws(IOException::class)
+    override fun generateNamespaceNd4j(namespace: NamespaceOps?, config: GeneratorConfig?, directory: File?, className: String?) {
+        val f = File(directory, GenUtil.ensureFirstIsCap(namespace!!.name) + ".py")
+        val content =
+        """
+            |class ${GenUtil.ensureFirstIsCap(namespace.name)}:
+            |${namespace.ops.filterNot { it.isAbstract }.joinToString("\n\n") { generateMethod(it).addIndent(4) }}
+        """.trimMargin()
+        FileUtils.writeStringToFile(f, content, StandardCharsets.UTF_8)
+    }
+
+    fun generateMethod(op: Op): String  =
+            """
+                |@staticmethod
+                |def ${GenUtil.ensureFirstIsNotCap(op.opName)}(${op.inputs.joinToString(", "){it.name}}):
+                |${genDocString(op).addIndent(4)}
+                |${"# Execution code here".addIndent(4)}
+                
+            """.trimMargin()
+
+    fun genDocString(op: Op): String {
+        //Following roughly: https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html
+        // python docstring / multiline string delimiter is the same as in kotlin, so use this little workaround
+        if (op.args.isNotEmpty()) {
+            //Args and default args
+            throw UnsupportedOperationException("Generating method with args not yet implemented")
+        }
+        if(op.outputs.size != 1) throw UnsupportedOperationException("Not yet implemented: Python docstring generation for multiple output ops")
+
+        val docStringDelimiter = "\"\"\""
+        return """
+                |$docStringDelimiter
+                |${op.opName} operation
+                |
+                |${op.inputs.let { """
+                |Args: 
+                |${it.joinToString("\n") {
+               "|    ${it.name} (ndarray): ${DocTokens.processDocText(it.description, op, DocTokens.GenerationType.ND4J)}"
+                }}
+                |""".trimMargin() }}
+                |${op.outputs.let {"""
+                |Returns:
+                |    ndarray: ${it[0].name} ${it[0].description?.let {"- ${DocTokens.processDocText(it, op, DocTokens.GenerationType.ND4J)}"
+                    }}""".trimMargin()
+                }}
+                |$docStringDelimiter
+            """.trimMargin()
+    }
+
+    @Throws(IOException::class)
+    override fun generateNamespaceSameDiff(namespace: NamespaceOps?, config: GeneratorConfig?, directory: File?, className: String?) {
+        throw UnsupportedOperationException("Not yet implemented")
+    }
+
+    private fun String.addIndent(size: Int): String = GenUtil.addIndent(this, size)
+}
+

codegen/op-codegen/src/main/kotlin/org/nd4j/codegen/util/GenerateOps.kt

@@ -0,0 +1,36 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.util
+
+import org.nd4j.codegen.impl.java.JavaPoetGenerator
+import org.nd4j.codegen.ops.Bitwise
+import org.nd4j.codegen.ops.Random
+import java.io.File
+
+fun main() {
+    val outDir = File("F:\\dl4j-builds\\deeplearning4j\\nd4j\\nd4j-backends\\nd4j-api-parent\\nd4j-api\\src\\main\\java\\")
+    outDir.mkdirs()
+
+    listOf(Bitwise(), Random()).forEach {
+        val generator = JavaPoetGenerator()
+        generator.generateNamespaceNd4j(it, null, outDir, it.name + ".java")
+    }
+}

codegen/op-codegen/src/main/ops/org/nd4j/codegen/mixins/Mixins.kt

@@ -0,0 +1,213 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.mixins
+
+import org.nd4j.codegen.api.AtLeast
+import org.nd4j.codegen.api.DataType
+import org.nd4j.codegen.api.Exactly
+import org.nd4j.codegen.api.Language
+import org.nd4j.codegen.api.doc.DocScope
+import org.nd4j.codegen.dsl.*
+
+val broadcastingDoc = Mixin("broadcastingDoc"){
+    Doc(Language.ANY, DocScope.ALL){
+        //TODO: finalize content for this broadcasting mixin doc.
+        """
+                Note: supports broadcasting if x and y have different shapes and are broadcastable.
+                For example, if X has shape [1,10] and Y has shape [5,10] then op(X,Y) has output shape [5,10]
+                Broadcast rules are the same as NumPy: https://docs.scipy.org/doc/numpy/user/basics.broadcasting.html
+            """.trimIndent()
+    }
+}
+
+val transform = Mixin("transform"){
+    legacy = true
+    Input(DataType.NUMERIC, "x") { description = "Input variable" }
+    Output(DataType.NUMERIC, "output"){ description = "Output variable" }
+}
+
+val transformArithmetic = Mixin("transformArithmetic"){
+    useMixin(transform)
+    legacy = false
+    Input(DataType.NUMERIC, "y") { description = "Input variable" }
+    javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic"
+}
+
+val transformCustom2 = Mixin("transformCustom2"){
+    Input(DataType.NUMERIC, "x") { description = "First input variable, x" }
+    Input(DataType.NUMERIC, "y") { description = "Second input variable, y" }
+    Output(DataType.NUMERIC, "out"){ description = "Output"}
+    javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+}
+
+val transformStrict = Mixin("transformStrict"){
+    useMixin(transform)
+    javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.strict"
+}
+
+val transformSame = Mixin("transformSame"){
+    useMixin(transform)
+    javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.same"
+}
+
+val transformBool = Mixin("transformBool"){
+    useMixin(transform)
+    javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.bool"
+}
+
+val transformAny = Mixin("transformAny"){
+    useMixin(transform)
+    javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.any"
+}
+
+val transformFloating = Mixin("transformFloating"){
+    useMixin(transform)
+    javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.floating"
+}
+
+val scalar = Mixin("scalar"){
+    legacy = true
+    javaPackage = "org.nd4j.linalg.api.ops.impl.scalar"
+    Input(DataType.NUMERIC, "x") { description = "Input variable" }
+    Arg(DataType.NUMERIC, "value") { description = "Scalar value for op" }
+    Output(DataType.NUMERIC, "output"){ description = "Output variable" }
+}
+
+val reduce = Mixin("reduce"){
+    Input(DataType.NUMERIC, "in") { description = "Input variable" }
+    Arg(DataType.BOOL,"keepDims"){ description = "Whether to keep the original  dimensions or produce a shrunk array with less dimensions"; defaultValue = false}
+    Arg(DataType.LONG, "dimensions"){  count = AtLeast(0); isVargarg = true;  description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed" }
+    Output(DataType.NUMERIC, "output"){ description = "Reduced array of rank (input rank - num dimensions)" }
+}
+
+val reduceVariableDimensions = Mixin("reduceVariable") {
+    Input(DataType.NUMERIC, "in") { description = "Input variable" }
+    Arg(DataType.BOOL,"keepDims"){ description = "Whether to keep the original  dimensions or produce a shrunk array with less dimensions"; defaultValue = false}
+    Input(DataType.NUMERIC, name = "dimensions"){description = "Dimensions to reduce along"; }
+    Output(DataType.NUMERIC, "output"){ description = "Reduced array of rank (input rank - num dimensions)" }
+}
+
+val reduceFloating = Mixin("reduceFloating"){
+    useMixin(reduce)
+    legacy = true
+    javaPackage = "org.nd4j.linalg.api.ops.impl.reduce.floating"
+}
+
+val reduceFloatingVariable = Mixin("reduceFloatingVariable"){
+    useMixin(reduceVariableDimensions)
+    legacy = true
+    javaPackage = "org.nd4j.linalg.api.ops.impl.reduce.floating"
+}
+
+
+val reduceSame = Mixin("reduceSame"){
+    useMixin(reduce)
+    legacy = true
+    javaPackage = "org.nd4j.linalg.api.ops.impl.reduce.same"
+}
+
+val reduceSameVariable = Mixin("reduceSameVariable"){
+    useMixin(reduceVariableDimensions)
+    legacy = true
+    javaPackage = "org.nd4j.linalg.api.ops.impl.reduce.same"
+}
+
+val reduceLong = Mixin("reduceLong"){
+    useMixin(reduce)
+    legacy = true
+    javaPackage = "org.nd4j.linalg.api.ops.impl.reduce.longer"
+}
+
+val reduceLongVariable = Mixin("reduceLongVariable"){
+    useMixin(reduceVariableDimensions)
+    javaPackage = "org.nd4j.linalg.api.ops.impl.reduce.longer"
+}
+
+val reduce3 = Mixin("reduce3"){
+    legacy = true
+    javaPackage = "org.nd4j.linalg.api.ops.impl.reduce3"
+    Input(DataType.NUMERIC, "x") { description = "Input variable x" }
+    Input(DataType.NUMERIC, "y") { description = "Input variable y" }
+    Arg(DataType.BOOL,"keepDims",{description = "Whether to preserve original dimensions or not"; defaultValue = false})
+    Arg(DataType.BOOL,"isComplex",{description = "Depending on the implementation, such as distance calculations, this can determine whether all distance calculations for all points should be done."; defaultValue = false})
+    val dims = Arg(DataType.LONG, "dimensions"){ count = AtLeast(0);  isVargarg = true; description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed" }
+    Output(DataType.NUMERIC, "output"){ description = "Output variable" }
+}
+
+val reduce3Variable = Mixin("reduce3Variable"){
+    legacy = true
+    javaPackage = "org.nd4j.linalg.api.ops.impl.reduce3"
+    Input(DataType.NUMERIC, "x") { description = "Input variable x" }
+    Input(DataType.NUMERIC, "y") { description = "Input variable y" }
+    Input(DataType.NUMERIC, "dimensions"){  description = "Dimensions to calculate %OPNAME% over" }
+    Output(DataType.NUMERIC, "output"){ description = "Output variable" }
+    Arg(DataType.BOOL,"keepDims",{description = "Whether to preserve original dimensions or not"; defaultValue = false})
+    Arg(DataType.BOOL,"isComplex",{description = "Depending on the implementation, such as distance calculations, this can determine whether all distance calculations for all points should be done."; defaultValue = false})
+
+}
+
+val indexAccum = Mixin("indexAccum"){
+    legacy = true
+    javaPackage = "org.nd4j.linalg.api.ops.impl.indexaccum"
+    val input = Input(DataType.NUMERIC, "in") { description = "Input variable" }
+    val keepDims = Arg(DataType.BOOL, "keepDims") { description = "If true: keep the dimensions that are reduced on (as length 1). False: remove the reduction dimensions"; defaultValue = false }
+    val dims = Arg(DataType.LONG, "dimensions"){ count = AtLeast(1); isVargarg = true; description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed" }
+    Output(DataType.NUMERIC, "output"){ description = "Reduced array of rank (input rank - num dimensions)" }
+
+    Signature(input, dims)
+    AllParamSignature(withOutput = false)
+}
+
+val indexAccumVariable = Mixin("indexAccumVariable"){
+    legacy = true
+    javaPackage = "org.nd4j.linalg.api.ops.impl.indexaccum"
+    val input = Input(DataType.NUMERIC, "in") { description = "Input variable" }
+    val keepDims = Arg(DataType.BOOL, "keepDims") { description = "If true: keep the dimensions that are reduced on (as length 1). False: remove the reduction dimensions"; defaultValue = false }
+    val dims = Input(DataType.NUMERIC, "dimensions"){  description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed" }
+    Output(DataType.NUMERIC, "output"){ description = "Reduced array of rank (input rank - num dimensions)" }
+
+    Signature(input, dims)
+    AllParamSignature(withOutput = false)
+}
+
+
+
+val indexAccumCustom = Mixin("indexAccumCustom"){
+    javaPackage = "org.nd4j.linalg.api.ops.impl.indexaccum.custom"
+    val input = Input(DataType.NUMERIC, "in") { description = "Input variable" }
+    val keepDims = Arg(DataType.BOOL, "keepDims") { description = "If true: keep the dimensions that are reduced on (as length 1). False: remove the reduction dimensions"; defaultValue = false }
+    val dims = Arg(DataType.LONG, "dimensions"){ count = AtLeast(1); isVargarg = true; description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed" }
+    Output(DataType.NUMERIC, "output"){ description = "Reduced array of rank (input rank - num dimensions)" }
+
+    Signature(input, dims)
+    AllParamSignature(withOutput = false)
+}
+
+val indexAccumVariableCustom = Mixin("indexAccumVariableCustom"){
+    javaPackage = "org.nd4j.linalg.api.ops.impl.indexaccum.custom"
+    val input = Input(DataType.NUMERIC, "in") { description = "Input variable" }
+    val keepDims = Arg(DataType.BOOL, "keepDims") { description = "If true: keep the dimensions that are reduced on (as length 1). False: remove the reduction dimensions"; defaultValue = false }
+    val dims = Input(DataType.NUMERIC, "dimensions"){  description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed" }
+    Output(DataType.NUMERIC, "output"){ description = "Reduced array of rank (input rank - num dimensions)" }
+
+    Signature(input, dims)
+    AllParamSignature(withOutput = false)
+}

codegen/op-codegen/src/main/ops/org/nd4j/codegen/ops/Bitwise.kt

@@ -0,0 +1,225 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.ops
+
+import org.nd4j.codegen.api.DataType.INT
+import org.nd4j.codegen.api.Language
+import org.nd4j.codegen.api.doc.DocScope
+import org.nd4j.codegen.dsl.*
+
+
+fun Bitwise() = Namespace("Bitwise"){
+    val namespaceJavaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+
+    Op("leftShift") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "ShiftBits"
+
+        Input(INT, "x") { description = "Input to be bit shifted" }
+        Input(INT, "y") { description = "Amount to shift elements of x array" }
+
+        Output(INT, "output"){ description = "Bitwise shifted input x" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+            Bitwise left shift operation. Supports broadcasting.
+            """.trimIndent()
+        }
+    }
+
+    Op("rightShift") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "RShiftBits"
+
+        Input(INT, "x") { description = "Input to be bit shifted" }
+        Input(INT, "y") { description = "Amount to shift elements of x array" }
+
+        Output(INT, "output"){ description = "Bitwise shifted input x" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+            Bitwise right shift operation. Supports broadcasting. 
+            """.trimIndent()
+        }
+    }
+
+    Op("leftShiftCyclic") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "CyclicShiftBits"
+
+        Input(INT, "x") { description = "Input to be bit shifted" }
+        Input(INT, "y") { description = "Amount to shift elements of x array" }
+
+        Output(INT, "output"){ description = "Bitwise cyclic shifted input x" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+            Bitwise left cyclical shift operation. Supports broadcasting.
+            Unlike #leftShift(%INPUT_TYPE%, %INPUT_TYPE%) the bits will "wrap around":
+            {@code leftShiftCyclic(01110000, 2) -> 11000001}
+            """.trimIndent()
+        }
+    }
+
+    Op("rightShiftCyclic") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "CyclicRShiftBits"
+
+        Input(INT, "x") { description = "Input to be bit shifted" }
+        Input(INT, "y") { description = "Amount to shift elements of x array" }
+
+        Output(INT, "output"){ description = "Bitwise cyclic shifted input x" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+            Bitwise right cyclical shift operation. Supports broadcasting.
+            Unlike rightShift(%INPUT_TYPE%, %INPUT_TYPE%) the bits will "wrap around":
+            {@code rightShiftCyclic(00001110, 2) -> 10000011}
+            """.trimIndent()
+        }
+    }
+
+    Op("bitsHammingDistance") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "BitsHammingDistance"
+
+        val x = Input(INT, "x") { description = "First input array." }
+        val y = Input(INT, "y") { description = "Second input array." }
+        Constraint("Must be same types"){ sameType(x, y) }
+
+        Output(INT, "output"){ description = "bitwise Hamming distance" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+            Bitwise Hamming distance reduction over all elements of both input arrays.<br>
+            For example, if x=01100000 and y=1010000 then the bitwise Hamming distance is 2 (due to differences at positions 0 and 1)
+            """.trimIndent()
+        }
+    }
+
+    Op("and") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "BitwiseAnd"
+
+        val x = Input(INT, "x") { description = "First input array" }
+        val y = Input(INT, "y") { description = "Second input array" }
+        Constraint("Must be same types"){ sameType(x, y) }
+        BackendConstraint("Must have broadcastable shapes"){ broadcastableShapes(x, y) }
+
+        Output(INT, "output"){ description = "Bitwise AND array" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+            Bitwise AND operation. Supports broadcasting.
+            """.trimIndent()
+        }
+    }
+
+    Op("or") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "BitwiseOr"
+
+        val x = Input(INT, "x") { description = "First input array" }
+        val y = Input(INT, "y") { description = "First input array" }
+        Constraint("Must be same types"){ sameType(x, y) }
+        BackendConstraint("Must have broadcastable shapes"){ broadcastableShapes(x, y) }
+
+        Output(INT, "output"){ description = "Bitwise OR array" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+            Bitwise OR operation. Supports broadcasting.
+            """.trimIndent()
+        }
+    }
+
+    Op("xor") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "BitwiseXor"
+
+        val x = Input(INT, "x") { description = "First input array" }
+        val y = Input(INT, "y") { description = "First input array" }
+        Constraint("Must be same types"){ sameType(x, y) }
+        BackendConstraint("Must have broadcastable shapes"){ broadcastableShapes(x, y) }
+
+        Output(INT, "output"){ description = "Bitwise XOR array" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+            Bitwise XOR operation (exclusive OR). Supports broadcasting.
+            """.trimIndent()
+        }
+    }
+
+    Op("bitShift") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "ShiftBits"
+        Input(INT, "x") { description = "Input 1" }
+        Input(INT, "shift") { description = "Number of bits to shift." }
+        Output(INT, "output"){ description = "SDVariable with shifted bits" }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                Shift integer bits to the left, i.e. var << 4
+            """.trimIndent()
+        }
+    }
+
+    Op("bitShiftRight") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "RShiftBits"
+        Input(INT, "x") { description = "Input 1" }
+        Input(INT, "shift") { description = "Number of bits to shift." }
+        Output(INT, "output"){ description = "SDVariable with shifted bits" }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                Shift integer bits to the right, i.e. var >> 4
+            """.trimIndent()
+        }
+    }
+
+    Op("bitRotl") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "CyclicShiftBits"
+        Input(INT, "x") { description = "Input 1" }
+        Input(INT, "shift") { description = "Number of bits to shift." }
+        Output(INT, "output"){ description = "SDVariable with shifted bits" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                Roll integer bits to the left, i.e. var << 4 | var >> (32 - 4)
+            """.trimIndent()
+        }
+    }
+
+    Op("bitRotr") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "CyclicRShiftBits"
+        Input(INT, "x") { description = "Input 1" }
+        Input(INT, "shift") { description = "Number of bits to shift." }
+        Output(INT, "output"){ description = "SDVariable with shifted bits" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                Roll integer bits to the right, i.e. var >> 4 | var << (32 - 4)
+            """.trimIndent()
+        }
+    }
+}

codegen/op-codegen/src/main/ops/org/nd4j/codegen/ops/CNN.kt

@@ -0,0 +1,586 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.ops
+
+import org.nd4j.codegen.api.AtLeast
+import org.nd4j.codegen.api.Language
+import org.nd4j.codegen.api.doc.DocScope
+import org.nd4j.codegen.dsl.*
+import org.nd4j.codegen.api.DataType.*
+import org.nd4j.codegen.api.Exactly
+
+fun SDCNN() =  Namespace("CNN"){
+    val namespaceJavaPackage = "org.nd4j.linalg.api.ops.impl.layers.convolution"
+
+    val dataFormat = Mixin("dataFormat"){
+        Arg(ENUM, "dataFormat") { possibleValues = listOf("NCHW", "NHWC"); description = "Data format: \"NCHW\" or \"NHWC\"" }
+    }
+
+
+    val conv1DConfig = Config("Conv1DConfig"){
+        Arg(LONG, "k"){ description = "Kernel"; defaultValue=-1L}
+        Arg(LONG, "s"){ description = "stride"; defaultValue=1}
+        Arg(LONG, "p"){ description = "padding"; defaultValue=0}
+        Arg(LONG, "d"){ description = "dilation"; defaultValue=1}
+        Arg(BOOL, "isSameMode"){ description = "Same mode"; defaultValue=true}
+        Arg(STRING, "dataFormat"){ description = "Data format"; defaultValue="NCW"}
+        javaClassOverride = "org.nd4j.linalg.api.ops.impl.layers.convolution.config.Conv1DConfig"
+    }
+
+    val conv2DConfig = Config("Conv2DConfig"){
+        Arg(LONG, "kH"){ description = "Kernel height"; defaultValue=-1L}
+        Arg(LONG, "kW"){ description = "Kernel width"; defaultValue=-1L}
+        Arg(LONG, "sH"){ description = "Stride along height dimension"; defaultValue=1};
+        Arg(LONG, "sW"){ description = "Stride along width dimension"; defaultValue=1};
+        Arg(LONG, "pH"){ description = "Padding along height dimension"; defaultValue=0};
+        Arg(LONG, "pW"){ description = "Padding along width dimension"; defaultValue=0};
+        Arg(LONG, "dH"){ description = "Dilation along height dimension"; defaultValue=1};
+        Arg(LONG, "dW"){ description = "Dilation along width dimension"; defaultValue=1};
+        Arg(BOOL, "isSameMode"){ description = "Same mode"; defaultValue=true}
+        Arg(STRING, "dataFormat"){ description = "Data format"; defaultValue="NCHW"}
+        javaClassOverride = "org.nd4j.linalg.api.ops.impl.layers.convolution.config.Conv2DConfig"
+    }
+
+    val conv3DConfig = Config("Conv3DConfig"){
+        Arg(LONG, "kD"){ description = "Kernel depth"; defaultValue=-1}
+        Arg(LONG, "kW"){ description = "Kernel width"; defaultValue=-1}
+        Arg(LONG, "kH"){ description = "Kernel height"; defaultValue=-1};
+        Arg(LONG, "sD"){ description = "Stride depth"; defaultValue=1};
+        Arg(LONG, "sW"){ description = "Stride width"; defaultValue=1};
+        Arg(LONG, "sH"){ description = "Stride height"; defaultValue=1};
+        Arg(LONG, "pD"){ description = "Padding depth"; defaultValue=0};
+        Arg(LONG, "pW"){ description = "Padding width"; defaultValue=0};
+        Arg(LONG, "pH"){ description = "Padding height"; defaultValue=0};
+        Arg(LONG, "dD"){ description = "Dilation depth"; defaultValue=1};
+        Arg(LONG, "dW"){ description = "Dilation width"; defaultValue=1};
+        Arg(LONG, "dH"){ description = "Dilation height"; defaultValue=1};
+        Arg(BOOL, "biasUsed"){ description = "biasUsed"; defaultValue=false}
+        Arg(BOOL, "isSameMode"){ description = "Same mode"; defaultValue=true}
+        Arg(STRING, "dataFormat"){ description = "Data format"; defaultValue="NDHWC"}
+        javaClassOverride = "org.nd4j.linalg.api.ops.impl.layers.convolution.config.Conv3DConfig"
+    }
+
+
+    val deconv2DConfig = Config("DeConv2DConfig"){
+        Arg(LONG, "kH"){ description = "Kernel height"; defaultValue=-1L}
+        Arg(LONG, "kW"){ description = "Kernel width"; defaultValue=-1L}
+        Arg(LONG, "sH"){ description = "Stride along height dimension"; defaultValue=1L};
+        Arg(LONG, "sW"){ description = "Stride along width dimension"; defaultValue=1L};
+        Arg(LONG, "pH"){ description = "Padding along height dimension"; defaultValue=0};
+        Arg(LONG, "pW"){ description = "Padding along width dimension"; defaultValue=0};
+        Arg(LONG, "dH"){ description = "Dilation along height dimension"; defaultValue=1L};
+        Arg(LONG, "dW"){ description = "Dilation along width dimension"; defaultValue=1L};
+        Arg(BOOL, "isSameMode"){ description = "Same mode"; defaultValue=false}
+        Arg(STRING, "dataFormat"){ description = "Data format"; defaultValue="NCHW"}
+        javaClassOverride = "org.nd4j.linalg.api.ops.impl.layers.convolution.config.DeConv2DConfig"
+    }
+
+
+    val deconv3DConfig = Config("DeConv3DConfig"){
+        Arg(LONG, "kD"){ description = "Kernel depth"; defaultValue=-1L}
+        Arg(LONG, "kW"){ description = "Kernel width"; defaultValue=-1L}
+        Arg(LONG, "kH"){ description = "Kernel height"; defaultValue=-1L};
+        Arg(LONG, "sD"){ description = "Stride depth"; defaultValue=1L};
+        Arg(LONG, "sW"){ description = "Stride width"; defaultValue=1L};
+        Arg(LONG, "sH"){ description = "Stride height"; defaultValue=1L};
+        Arg(LONG, "pD"){ description = "Padding depth"; defaultValue=0};
+        Arg(LONG, "pW"){ description = "Padding width"; defaultValue=0};
+        Arg(LONG, "pH"){ description = "Padding height"; defaultValue=0};
+        Arg(LONG, "dD"){ description = "Dilation depth"; defaultValue=1L};
+        Arg(LONG, "dW"){ description = "Dilation width"; defaultValue=1L};
+        Arg(LONG, "dH"){ description = "Dilation height"; defaultValue=1L};
+        Arg(BOOL, "isSameMode"){ description = "Same mode"; defaultValue=false}
+        Arg(STRING, "dataFormat"){ description = "Data format"; defaultValue="NCDHW"}
+        javaClassOverride = "org.nd4j.linalg.api.ops.impl.layers.convolution.config.DeConv3DConfig"
+    }
+
+
+
+
+    val pooling2DConfig = Config("Pooling2DConfig"){
+        Arg(LONG, "kH"){ description = "Kernel height"; defaultValue=-1}
+        Arg(LONG, "kW"){ description = "Kernel width"; defaultValue=-1}
+        Arg(LONG, "sH"){ description = "Stride along height dimension"; defaultValue=1};
+        Arg(LONG, "sW"){ description = "Stride along width dimension"; defaultValue=1};
+        Arg(LONG, "pH"){ description = "Padding along height dimension"; defaultValue=0};
+        Arg(LONG, "pW"){ description = "Padding along width dimension"; defaultValue=0};
+        Arg(LONG, "dH"){ description = "Dilation along height dimension"; defaultValue=1};
+        Arg(LONG, "dW"){ description = "Dilation along width dimension"; defaultValue=1};
+        Arg(BOOL, "isSameMode"){ description = "Same mode"; defaultValue=true}
+        Arg(STRING, "dataFormat"){ description = "Data format"; defaultValue="nchw"}
+        javaClassOverride = "org.nd4j.linalg.api.ops.impl.layers.convolution.config.Pooling2DConfig"
+    }
+
+    val pooling3DConfig = Config("Pooling3DConfig"){
+        Arg(LONG, "kD"){ description = "Kernel depth"; defaultValue=-1}
+        Arg(LONG, "kW"){ description = "Kernel width"; defaultValue=-1}
+        Arg(LONG, "kH"){ description = "Kernel height"; defaultValue=-1};
+        Arg(LONG, "sD"){ description = "Stride depth"; defaultValue=1};
+        Arg(LONG, "sW"){ description = "Stride width"; defaultValue=1};
+        Arg(LONG, "sH"){ description = "Stride height"; defaultValue=1};
+        Arg(LONG, "pD"){ description = "Padding depth"; defaultValue=0};
+        Arg(LONG, "pW"){ description = "Padding width"; defaultValue=0};
+        Arg(LONG, "pH"){ description = "Padding height"; defaultValue=0};
+        Arg(LONG, "dD"){ description = "Dilation depth"; defaultValue=1};
+        Arg(LONG, "dW"){ description = "Dilation width"; defaultValue=1};
+        Arg(LONG, "dH"){ description = "Dilation height"; defaultValue=1};
+        Arg(BOOL, "isSameMode"){ description = "Same mode"; defaultValue=true}
+        Arg(STRING, "dataFormat"){ description = "Data format"; defaultValue="NCDHW"}
+        javaClassOverride = "org.nd4j.linalg.api.ops.impl.layers.convolution.config.Pooling3DConfig"
+    }
+
+
+    val LocalResponseNormalizationConfig = Config("LocalResponseNormalizationConfig"){
+        Arg(NUMERIC, "alpha"){ description = "alpha"; defaultValue=1}
+        Arg(NUMERIC, "beta"){ description = "beta"; defaultValue=0.5}
+        Arg(NUMERIC, "bias"){ description = "bias"; defaultValue=1}
+        Arg(INT, "depth"){ description = "depth"; defaultValue=5}
+        javaClassOverride = "org.nd4j.linalg.api.ops.impl.layers.convolution.config.LocalResponseNormalizationConfig"
+
+
+    }
+
+
+
+
+    Op("avgPooling2d") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "AvgPooling2D"
+        Input(NUMERIC, "input") { description = "the input to average pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels])" }
+        useConfig(pooling2DConfig)
+
+        Output(NUMERIC, "output"){ description = "Result after applying average pooling on the input" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             2D Convolution layer operation - average pooling 2d
+            """.trimIndent()
+        }
+    }
+
+    Op("avgPooling3d") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "AvgPooling3D"
+        Input(NUMERIC, "input") {description = "the input to average pooling 3d operation - 5d activations in NCDHW format (shape [minibatch, channels, depth, height, width]) or NDHWC format (shape [minibatch, depth, height, width, channels])" }
+        useConfig(pooling3DConfig)
+
+        Output(NUMERIC, "output"){ description = "after applying average pooling on the input" }
+
+        Doc(Language.ANY, DocScope.ALL){
+        """
+         3D convolution layer operation - average pooling 3d 
+        """.trimIndent()
+        }
+    }
+
+    Op("batchToSpace") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "BatchToSpace"
+        Input(NUMERIC, "x") { description = "Input variable. 4d input" }
+        Arg(INT, "blocks") { count=Exactly(2); description = "Block size, in the height/width dimension" }
+        Arg(INT, "croppingTop") { count=Exactly(2)}
+        Arg(INT, "croppingBottom") { count=Exactly(2)}
+        Output(NUMERIC, "output"){ description = "Output variable" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Convolution 2d layer batch to space operation on 4d input.
+             Reduces input batch dimension by rearranging data into a larger spatial dimensions
+            """.trimIndent()
+        }
+    }
+
+    Op("col2Im") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "Col2Im"
+
+        Input(NUMERIC, "in") { description = "Input - rank 6 input with shape [minibatch, inputChannels, kernelHeight, kernelWidth, outputHeight, outputWidth]" }
+        useConfig(conv2DConfig)
+
+        Output(NUMERIC, "output"){ description = "Col2Im output variable" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             col2im operation for use in 2D convolution operations. Outputs a 4d array with shape
+             [minibatch, inputChannels, height, width]
+            """.trimIndent()
+        }
+    }
+
+
+
+    Op("conv1d") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "Conv1D"
+        Input(NUMERIC, "input") { description = "the inputs to conv1d" }
+        Input(NUMERIC, "weights") { description = "weights for conv1d op - rank 3 array with shape [kernelSize, inputChannels, outputChannels]" }
+        Input(NUMERIC, "bias") { description = "bias for conv1d op - rank 1 array with shape [outputChannels]. May be null."; defaultValue=null }
+        useConfig(conv1DConfig)
+
+        Output(NUMERIC, "output"){ description = "result of conv1d op" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Conv1d operation.
+            """.trimIndent()
+        }
+    }
+
+
+
+    Op("conv2d") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "Conv2D"
+        Input(NUMERIC, "layerInput") { description = "the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format" }
+        Input(NUMERIC, "weights") { description = "Weights for the convolution operation. 4 dimensions with format [kernelHeight, kernelWidth, inputChannels, outputChannels]" }
+        Input(NUMERIC, "bias") { description = "Optional 1D bias array with shape [outputChannels]. May be null."; defaultValue=null }
+        useConfig(conv2DConfig)
+
+        Output(NUMERIC, "output"){ description = "result of conv2d op" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             2D Convolution operation with optional bias
+            """.trimIndent()
+        }
+    }
+
+
+
+
+    Op("conv3d") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "Conv3D"
+        Input(NUMERIC, "input") { description = "the input to average pooling 3d operation - 5d activations in NCDHW format (shape [minibatch, channels, depth, height, width]) or NDHWC format (shape [minibatch, depth, height, width, channels])" }
+        Input(NUMERIC, "weights") { description = " Weights for conv3d. Rank 5 with shape [kernelDepth, kernelHeight, kernelWidth, inputChannels, outputChannels]." }
+        Input(NUMERIC, "bias") { description = " Optional 1D bias array with shape [outputChannels]. May be null."; defaultValue=null }
+        useConfig(conv3DConfig)
+
+        Output(NUMERIC, "output"){ description = "Conv3d output variable" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Convolution 3D operation with optional bias 
+            """.trimIndent()
+        }
+    }
+
+
+
+    Op("deconv2d") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "DeConv2D"
+        Input(NUMERIC, "layerInput") { description = "the input to deconvolution 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels])" }
+        Input(NUMERIC, "weights") { description = "Weights for the 2d deconvolution operation. 4 dimensions with format [inputChannels, outputChannels, kernelHeight, kernelWidth]" }
+        Input(NUMERIC, "bias") { description = "Optional 1D bias array with shape [outputChannels]. May be null."; defaultValue=null }
+        useConfig(deconv2DConfig)
+        Output(NUMERIC, "output"){ description = "result of deconv2d op" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             2D deconvolution operation with optional bias
+            """.trimIndent()
+        }
+    }
+
+
+
+
+
+    Op("deconv3d") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "DeConv3D"
+        Input(NUMERIC, "input") { description = "Input array - shape [bS, iD, iH, iW, iC] (NDHWC) or [bS, iC, iD, iH, iW] (NCDHW)" }
+        Input(NUMERIC, "weights") { description = "Weights array - shape [kD, kH, kW, oC, iC]" }
+        Input(NUMERIC, "bias") { description = "Bias array - optional, may be null. If non-null, must have shape [outputChannels]"; defaultValue=null }
+        useConfig(deconv3DConfig)
+
+        Output(NUMERIC, "output"){ description = "result of 3D CNN deconvolution operation" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             3D CNN deconvolution operation with or without optional bias
+            """.trimIndent()
+        }
+    }
+
+    Op("depthToSpace") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "DepthToSpace"
+        Input(NUMERIC, "x") { description = "the input to depth to space pooling 2d operation - 4d activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels])" }
+        Arg(INT, "blockSize") { description = "Block size, in the height/width dimension" }
+        useMixin(dataFormat)
+        Output(NUMERIC, "output"){ description = "Output variable" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Convolution 2d layer batch to space operation on 4d input.<br>
+             Reduces input channels dimension by rearranging data into a larger spatial dimensions<br>
+             Example: if input has shape [mb, 8, 2, 2] and block size is 2, then output size is [mb, 8/(2*2), 2*2, 2*2]
+             = [mb, 2, 4, 4]
+            """.trimIndent()
+        }
+    }
+
+
+    Op("depthWiseConv2d") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "DepthwiseConv2D"
+        Input(NUMERIC, "layerInput") { description = "the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format" }
+        Input(NUMERIC, "depthWeights") { description = "Depth-wise conv2d weights. 4 dimensions with format [kernelHeight, kernelWidth, inputChannels, depthMultiplier]" }
+        Input(NUMERIC, "bias") { description = "Optional 1D bias array with shape [outputChannels]. May be null."; defaultValue=null }
+        useConfig(conv2DConfig)
+
+        Output(NUMERIC, "output"){ description = "result of depthwise conv2d op" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Depth-wise 2D convolution operation with optional bias 
+            """.trimIndent()
+        }
+    }
+
+
+
+    Op("dilation2D") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "Dilation2D"
+        Input(NUMERIC, "df") { description = "" }
+        Input(NUMERIC, "weights") { description = "df" }
+        Arg(INT, "strides") { count = Exactly(2); description = "weights" }
+        Arg(INT, "rates") {count = Exactly(2); description = "strides" }
+        Arg(BOOL, "isSameMode") { description = "isSameMode" }
+
+        Output(NUMERIC, "output"){ description = "Computed the grayscale dilation of 4-D input and 3-D filters tensors." }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             TODO doc string
+            """.trimIndent()
+        }
+    }
+
+    Op("extractImagePatches") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.image"
+        javaOpClass = "ExtractImagePatches"
+        Input(NUMERIC, "input") { description = "Input array. Must be rank 4, with shape [minibatch, height, width, channels]" }
+        Arg(INT, "kH") { description = "Kernel height" }
+        Arg(INT, "kW") { description = "Kernel width" }
+        Arg(INT, "sH") { description = "Stride height" }
+        Arg(INT, "sW") { description = "Stride width" }
+        Arg(INT, "rH") { description = "Rate height" }
+        Arg(INT, "rW") { description = "Rate width" }
+        Arg(BOOL, "sameMode") { description = "If true: use same mode padding. If false" }
+
+        Output(NUMERIC, "output"){ description = "The result is a 4D tensor which is indexed by batch, row, and column." }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Extract image patches 
+            """.trimIndent()
+        }
+    }
+
+    Op("im2Col") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "Im2col"
+        Input(NUMERIC, "in") { description = "Input - rank 4 input with shape [minibatch, inputChannels, height, width]" }
+        useConfig(conv2DConfig)
+
+        Output(NUMERIC, "output"){ description = "Im2Col output variable" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             im2col operation for use in 2D convolution operations. Outputs a 6d array with shape
+             [minibatch, inputChannels, kernelHeight, kernelWidth, outputHeight, outputWidth]   
+            """.trimIndent()
+        }
+    }
+
+    Op("localResponseNormalization") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "LocalResponseNormalization"
+        Input(NUMERIC, "input") { description = "the inputs to lrn" }
+        useConfig(LocalResponseNormalizationConfig)
+
+        Output(NUMERIC, "output"){ description = "Result after Local Response Normalization"}
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             2D convolution layer operation - local response normalization
+            """.trimIndent()
+        }
+    }
+
+    Op("maxPooling2d") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "MaxPooling2D"
+        Input(NUMERIC, "input") { description = "the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels])" }
+        useConfig(pooling2DConfig)
+
+        Output(NUMERIC, "output"){ description = "Result after applying max pooling on the input" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             2D Convolution layer operation - max pooling 2d 
+            """.trimIndent()
+        }
+    }
+
+    Op("maxPoolWithArgmax") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "MaxPoolWithArgmax"
+        Input(NUMERIC, "input") { description = "the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels])" }
+        useConfig(pooling2DConfig)
+
+        Output(NUMERIC, "output"){ description = "Result after applying max pooling on the input" }
+        Output(NUMERIC, "indexes"){ description = "Argmax array" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             2D Convolution layer operation - Max pooling on the input and outputs both max values and indices 
+            """.trimIndent()
+        }
+    }
+
+    Op("maxPooling3d") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "MaxPooling3D"
+        Input(NUMERIC, "input") { description = "the input to average pooling 3d operation - 5d activations in NCDHW format (shape [minibatch, channels, depth, height, width]) or NDHWC format (shape [minibatch, depth, height, width, channels])" }
+        useConfig(pooling3DConfig)
+
+        Output(NUMERIC, "output"){ description = "Result after applying max pooling on the input" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             3D convolution layer operation - max pooling 3d operation.
+            """.trimIndent()
+        }
+    }
+
+
+
+    Op("separableConv2d") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.layers.convolution"
+        javaOpClass = "SConv2D"
+        Input(NUMERIC, "layerInput") { description = "the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels])" }
+        Input(NUMERIC, "depthWeights") { description = "Separable conv2d depth weights. 4 dimensions with format [kernelHeight, kernelWidth, inputChannels, depthMultiplier]" }
+        Input(NUMERIC, "pointWeights") { description = "Point weights, rank 4 with format [1, 1, inputChannels*depthMultiplier, outputChannels]. May be null" }
+        Input(NUMERIC, "bias") { description = "Optional bias, rank 1 with shape [outputChannels]. May be null."; defaultValue=null}
+        useConfig(conv2DConfig)
+
+        Output(NUMERIC, "output"){ description = "result of separable convolution 2d operation" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Separable 2D convolution operation with optional bias 
+            """.trimIndent()
+        }
+    }
+
+
+
+    Op("spaceToBatch") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "SpaceToBatch"
+        Input(NUMERIC, "x") { description = "Input variable. 4d input" }
+        Arg(INT, "blocks") { count = Exactly(2); description = "Block size, in the height/width dimension" }
+        Arg(INT, "paddingTop") {count = Exactly(2); description = "Optional 2d int[] array for padding the result: values [[pad top, pad bottom], [pad left, pad right]]" }
+        Arg(INT, "paddingBottom") {count = Exactly(2); description = "Optional 2d int[] array for padding the result: values [[pad top, pad bottom], [pad left, pad right]]" }
+
+        Output(NUMERIC, "output"){ description = "Output variable" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Convolution 2d layer space to batch operation on 4d input.
+             Increases input batch dimension by rearranging data from spatial dimensions into batch dimension 
+            """.trimIndent()
+        }
+    }
+
+    Op("spaceToDepth") {
+        javaPackage = namespaceJavaPackage
+        Input(NUMERIC, "x") { description = "the input to depth to space pooling 2d operation - 4d activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels])" }
+        Arg(INT, "blockSize") { description = " Block size, in the height/width dimension" }
+        useMixin(dataFormat)
+        Output(NUMERIC, "output"){ description = "Output variable" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Convolution 2d layer space to depth operation on 4d input.<br>
+             Increases input channels (reduced spatial dimensions) by rearranging data into a larger channels dimension<br>
+             Example: if input has shape [mb, 2, 4, 4] and block size is 2, then output size is [mb, 8/(2*2), 2*2, 2*2]
+             = [mb, 2, 4, 4] 
+            """.trimIndent()
+        }
+    }
+
+    Op("upsampling2d") {
+        javaPackage = namespaceJavaPackage
+        Input(NUMERIC, "input") { description = "Input in NCHW format" }
+        Arg(INT, "scale") { description = "The scale for both height and width dimensions." }
+
+        Output(NUMERIC, "output"){ description = "Upsampled input"}
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Upsampling layer for 2D inputs.
+             scale is used for both height and width dimensions. 
+            """.trimIndent()
+        }
+    }
+
+    Op("upsampling2d") {
+        javaPackage = namespaceJavaPackage
+        Input(NUMERIC, "input") { description = "Input in NCHW format" }
+        Arg(INT, "scaleH") { description = "Scale to upsample in height dimension" }
+        Arg(INT, "scaleW") { description = "Scale to upsample in width dimension" }
+        Arg(BOOL ,"nchw") { description = "If true: input is in NCHW (minibatch, channels, height, width) format. False: NHWC format" }
+
+
+        Output(NUMERIC, "output"){ description = "Upsampled input" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             2D Convolution layer operation - Upsampling 2d 
+            """.trimIndent()
+        }
+    }
+
+    Op("upsampling3d") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "Upsampling3d"
+        Input(NUMERIC, "input") { description = "Input in NCHW format" }
+        Arg(BOOL ,"ncdhw") { description = "If true: input is in NCDHW (minibatch, channels, depth, height, width) format. False: NDHWC format" }
+        Arg(INT, "scaleD") { description = "Scale to upsample in depth dimension" }
+        Arg(INT, "scaleH") { description = "Scale to upsample in height dimension" }
+        Arg(INT, "scaleW") { description = "Scale to upsample in width dimension" }
+
+
+        Output(NUMERIC, "output"){ description = "Upsampled input" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             3D Convolution layer operation - Upsampling 3d 
+            """.trimIndent()
+        }
+    }
+}

codegen/op-codegen/src/main/ops/org/nd4j/codegen/ops/Image.kt

@@ -0,0 +1,313 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.ops
+
+import org.nd4j.codegen.api.AtLeast
+import org.nd4j.codegen.api.Language
+import org.nd4j.codegen.api.doc.DocScope
+import org.nd4j.codegen.dsl.*
+import org.nd4j.codegen.api.DataType.*
+import org.nd4j.codegen.api.Exactly
+
+
+fun SDImage() =  Namespace("Image"){
+    val namespaceJavaPackage = "org.nd4j.linalg.api.ops.custom"
+    Op("CropAndResize") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.image"
+        javaOpClass = "CropAndResize"
+        Input(NUMERIC, "image") { description = "Input image, with shape [batch, height, width, channels]" }
+        Input(NUMERIC, "cropBoxes") { description = "Float32 crop, shape [numBoxes, 4] with values in range 0 to 1" }
+        Input(NUMERIC, "boxIndices") { description = "Indices: which image (index to dimension 0) the cropBoxes belong to. Rank 1, shape [numBoxes]" }
+        Input(INT, "cropOutSize") { description = "Output size for the images - int32, rank 1 with values [outHeight, outWidth]" }
+        Arg(NUMERIC, "extrapolationValue") { description = "Used for extrapolation, when applicable. 0.0 should be used for the default"; defaultValue=0.0 }
+
+        Output(NUMERIC, "output"){ description = "Cropped and resized images" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Given an input image and some crop boxes, extract out the image subsets and resize them to the specified size.
+            """.trimIndent()
+        }
+    }
+
+    Op("extractImagePatches") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.image"
+        javaOpClass = "ExtractImagePatches"
+        Input(NUMERIC, "image") { description = "Input image to extract image patches from - shape [batch, height, width, channels]" }
+        Arg(INT, "kSizes") { count = Exactly(2); description = "Kernel size - size of the image patches, [height, width]" }
+        Arg(INT, "strides") { count = Exactly(2);description = "Stride in the input dimension for extracting image patches, [stride_height, stride_width]" }
+        Arg(INT, "rates") { count = AtLeast(0); description = "Usually [1,1]. Equivalent to dilation rate in dilated convolutions - how far apart the output pixels\n" +
+                "                 in the patches should be, in the input. A dilation of [a,b] means every {@code a}th pixel is taken\n" +
+                "                 along the height/rows dimension, and every {@code b}th pixel is take along the width/columns dimension" }
+        Arg(BOOL, "sameMode") { description = "Padding algorithm. If true: use Same padding" }
+
+        Output(NUMERIC, "output"){ description = "The extracted image patches" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Given an input image, extract out image patches (of size kSizes - h x w) and place them in the depth dimension. 
+            """.trimIndent()
+        }
+    }
+
+    Op("nonMaxSuppression") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.image"
+        javaOpClass = "NonMaxSuppression"
+        Input(NUMERIC, "boxes") { description = "Might be null. Name for the output variable" }
+        Input(NUMERIC, "scores") { description = "vector of shape [num_boxes]" }
+        Arg(INT, "maxOutSize") { description = "scalar representing the maximum number of boxes to be selected" }
+        Arg(NUMERIC, "iouThreshold") { description = "threshold for deciding whether boxes overlap too much with respect to IOU" }
+        Arg(NUMERIC, "scoreThreshold") { description = "threshold for deciding when to remove boxes based on score" }
+
+        Output(NUMERIC, "output"){ description = "vectort of shape [M] representing the selected indices from the boxes tensor, where M <= max_output_size" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+            Greedily selects a subset of bounding boxes in descending order of score
+            """.trimIndent()
+        }
+    }
+
+    Op("adjustContrast") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "AdjustContrast"
+        Input(NUMERIC, "in") { description = "images to adjust. 3D shape or higher" }
+        Arg(FLOATING_POINT, "factor") { description = "multiplier for adjusting contrast" }
+
+        Output(NUMERIC, "output"){ description = "Contrast-adjusted image" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Adjusts contrast of RGB or grayscale images.
+            """.trimIndent()
+        }
+    }
+
+    Op("adjustSaturation") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "AdjustSaturation"
+        Input(NUMERIC, "in") { description = "RGB image as 3D array" }
+        Arg(FLOATING_POINT, "factor") { description = "factor for saturation" }
+
+        Output(NUMERIC, "output"){ description = "adjusted image" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Adjust saturation of RGB images
+            """.trimIndent()
+        }
+    }
+
+    Op("adjustHue") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "AdjustHue"
+        Input(NUMERIC, "in") { description = "image as 3D array" }
+        Arg(NUMERIC, "delta") { description = "value to add to hue channel" }
+
+        Output(NUMERIC, "output"){ description = "adjusted image" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Adjust hue of RGB image 
+            """.trimIndent()
+        }
+    }
+
+
+    Op("pad") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms"
+        javaOpClass = "Pad"
+        Input(NUMERIC, "input") { description = "input array" }
+        Input(NUMERIC, "padding") { description = "padding input" }
+        Arg(ENUM,"Mode") {possibleValues = listOf("CONSTANT", "REFLECT", "SYMMETRIC"); description = "padding mode: CONSTANT, REFLECT, SYMMETRIC"}
+        Arg(NUMERIC, "padValue") { description = "The value to pad with" }
+
+        Output(NUMERIC, "output"){ description = "the padded array" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Pads an image according to the given padding type
+            """.trimIndent()
+        }
+    }
+
+
+    Op("randomCrop") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "RandomCrop"
+        Input(NUMERIC, "input") { description = "input array" }
+        Input(INT, "shape") { description = "shape for crop" }
+
+        Output(NUMERIC, "output"){ description = "cropped array" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Randomly crops image
+            """.trimIndent()
+        }
+    }
+
+    Op("rgbToHsv") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "RgbToHsv"
+        Input(NUMERIC, "input") { description = "3D image" }
+
+        Output(NUMERIC, "output"){ description = "3D image" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Converting array from HSV to RGB format
+            """.trimIndent()
+        }
+    }
+
+    Op("hsvToRgb") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "HsvToRgb"
+        Input(NUMERIC, "input") { description = "3D image" }
+
+        Output(NUMERIC, "output"){ description = "3D image" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Converting image from HSV to RGB format 
+            """.trimIndent()
+        }
+    }
+
+    Op("rgbToYiq") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "RgbToYiq"
+        Input(NUMERIC, "input") { description = "3D image" }
+
+        Output(NUMERIC, "output"){ description = "3D image" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Converting array from RGB to YIQ format 
+            """.trimIndent()
+        }
+    }
+
+    Op("yiqToRgb") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "YiqToRgb"
+        Input(NUMERIC, "input") { description = "3D image" }
+
+        Output(NUMERIC, "output"){ description = "3D image" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Converting image from YIQ to RGB format 
+            """.trimIndent()
+        }
+    }
+
+    Op("rgbToYuv") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "RgbToYuv"
+
+        Input(NUMERIC, "input") { description = "3D image" }
+
+        Output(NUMERIC, "output"){ description = "3D image" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Converting array from RGB to YUV format 
+            """.trimIndent()
+        }
+    }
+
+    Op("yuvToRgb") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "YuvToRgb"
+
+        Input(NUMERIC, "input") { description = "3D image" }
+
+        Output(NUMERIC, "output"){ description = "3D image" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Converting image from YUV to RGB format 
+            """.trimIndent()
+        }
+    }
+
+    Op("imageResize") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.image"
+        javaOpClass = "ImageResize"
+
+        Input(NUMERIC, "input") { description = "4D image [NHWC]" }
+        Input(INT, "size") { description = "new height and width" }
+        Arg(BOOL, "preserveAspectRatio") { description = "Whether to preserve the aspect ratio." +
+                " If this is set, then images will be resized to a size that fits in size while preserving the aspect ratio" +
+                " of the original image. Scales up the image if size is bigger than the current size of the image. Defaults to False."; defaultValue=false; }
+        Arg(BOOL, "antialias") { description = "Whether to use an anti-aliasing filter when downsampling an image"; defaultValue=false;  }
+        Arg(ENUM, "ImageResizeMethod") { possibleValues = listOf( "ResizeBilinear", "ResizeBicubic", "ResizeNearest", "ResizeGaussian",
+                "ResizeLanczos5", "ResizeMitchellcubic", "ResizeArea"); description = "ResizeBilinear: Bilinear interpolation. If 'antialias' is true, becomes a hat/tent filter function with radius 1 when downsampling.\n" +
+                "ResizeLanczos5: Lanczos kernel with radius 5. Very-high-quality filter but may have stronger ringing.\n" +
+                "ResizeBicubic: Cubic interpolant of Keys. Equivalent to Catmull-Rom kernel. Reasonably good quality and faster than Lanczos3Kernel, particularly when upsampling.\n" +
+                "ResizeGaussian: Gaussian kernel with radius 3, sigma = 1.5 / 3.0.\n" +
+                "ResizeNearest: Nearest neighbor interpolation. 'antialias' has no effect when used with nearest neighbor interpolation.\n" +
+                "ResizeArea: Anti-aliased resampling with area interpolation. 'antialias' has no effect when used with area interpolation; it always anti-aliases.\n" +
+                "ResizeMitchellcubic: Mitchell-Netravali Cubic non-interpolating filter. For synthetic images (especially those lacking proper prefiltering), less ringing than Keys cubic kernel but less sharp." }
+
+        Output(NUMERIC, "output"){ description = "Output image" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Resize images to size using the specified method.
+            """.trimIndent()
+        }
+    }
+
+    Op("resizeBiLinear") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.image"
+        javaOpClass = "ResizeBilinear"
+        Input(NUMERIC,"input") { description = "4D image"}
+        Arg(INT ,"height") { description = "target height for resizing to "}
+        Arg(INT ,"width") { description = "target width for resizing to"}
+        Arg(BOOL ,"alignCorners") { description = "whether to align corners during resizing. Images are aligned to preserve corners."}
+        Arg(BOOL,"halfPixelCenters") { description = "When resizing, assumes pixels are centered at 0.5."}
+        Output(NUMERIC, "output"){ description = "Output image" }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Resize images to size using the specified method.
+            """.trimIndent()
+        }
+    }
+
+    Op("resizeBiCubic") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.image"
+        javaOpClass = "ResizeBicubic"
+        Input(NUMERIC,"input") { description = "4D image"}
+        Input(INT ,"size") { description = "the target size to resize to "}
+        Arg(BOOL ,"alignCorners") { description = "whether to align corners during resizing. Images are aligned to preserve corners."}
+        Arg(BOOL,"alignPixelCenters") { description = "When resizing, assumes pixels are centered at 0.5."}
+        Output(NUMERIC, "output"){ description = "Output image" }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Resize images to size using the specified method.
+            """.trimIndent()
+        }
+    }
+
+}

codegen/op-codegen/src/main/ops/org/nd4j/codegen/ops/Linalg.kt

@@ -0,0 +1,322 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.ops
+
+import org.nd4j.codegen.api.DataType
+import org.nd4j.codegen.api.DataType.*
+import org.nd4j.codegen.api.Language
+import org.nd4j.codegen.api.doc.DocScope
+import org.nd4j.codegen.dsl.*
+import org.nd4j.codegen.api.Range
+
+
+fun Linalg() =  Namespace("Linalg") {
+    //val namespaceJavaPackage = "org.nd4j.linalg"
+
+    Op("Cholesky") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms"
+        javaOpClass = "Cholesky"
+        Input(DataType.NUMERIC, "input") { description = "Input tensor with inner-most 2 dimensions forming square matrices" }
+        Output(DataType.NUMERIC, "output"){ description = "Transformed tensor" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Computes the Cholesky decomposition of one or more square matrices.
+            """.trimIndent()
+        }
+    }
+
+    Op("Lstsq") {
+        javaPackage = "org.nd4j.linalg.api.ops.custom"
+        javaOpClass = "Lstsq"
+
+        Input(DataType.NUMERIC, "matrix") {description = "input tensor"}
+        Input(DataType.NUMERIC, "rhs") {description = "input tensor"}
+        Arg(DataType.FLOATING_POINT, "l2_reguralizer") {description = "regularizer"}
+        Arg(DataType.BOOL, "fast") {description = "fast mode, defaults to True"; defaultValue = true}
+        Output(DataType.FLOATING_POINT, "output"){ description = "Transformed tensor" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Solver for linear squares problems.
+            """.trimIndent()
+        }
+    }
+
+    Op("Solve") {
+        javaPackage = "org.nd4j.linalg.api.ops.custom"
+        javaOpClass = "LinearSolve"
+
+        Input(DataType.NUMERIC, "matrix") {description = "input tensor"}
+        Input(DataType.NUMERIC, "rhs") {description = "input tensor"}
+        Arg(DataType.BOOL, "adjoint") {description = "adjoint mode, defaults to False"; defaultValue = false}
+        Output(FLOATING_POINT, "output"){ description = "Output tensor" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Solver for systems of linear equations.
+            """.trimIndent()
+        }
+    }
+
+    Op("TriangularSolve") {
+        javaPackage = "org.nd4j.linalg.api.ops.custom"
+        javaOpClass = "TriangularSolve"
+
+        Input(DataType.NUMERIC, "matrix") {description = "input tensor"}
+        Input(DataType.NUMERIC, "rhs") {description = "input tensor"}
+        Arg(DataType.BOOL, "lower") {description = "defines whether innermost matrices in matrix are lower or upper triangular"}
+        Arg(DataType.BOOL, "adjoint") {description = "adjoint mode"}
+        Output(DataType.FLOATING_POINT, "output")
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Solver for systems of linear questions.
+            """.trimIndent()
+        }
+    }
+
+    Op("Lu") {
+        javaPackage = "org.nd4j.linalg.api.ops.custom"
+        javaOpClass = "Lu"
+
+        Input(DataType.NUMERIC, "input") {description = "input tensor"}
+        Output(FLOATING_POINT, "output")
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Computes LU decomposition.
+            """.trimIndent()
+        }
+    }
+
+    Op("Matmul") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.reduce"
+        javaOpClass = "Mmul"
+
+        Input(DataType.NUMERIC, "a") {description = "input tensor"}
+        Input(DataType.NUMERIC, "b") {description = "input tensor"}
+        Arg(DataType.FLOATING_POINT,"alpha",{defaultValue = 1.0; description = "Defaults to 1.0: the scalar multiplier for the product of a* b "})
+        Arg(DataType.FLOATING_POINT,"beta",{defaultValue = 1.0; description = "Defaults to 1.0: the scalar multiplier for c "})
+        Arg(DataType.BOOL,"transA",{defaultValue = false; description = "Whether to transpose a when running multiply "})
+        Arg(DataType.BOOL,"transB",{defaultValue = false; description = "Whether to transpose b when running multiply "})
+        Output(DataType.FLOATING_POINT, "output")
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Performs matrix multiplication on input tensors.
+            """.trimIndent()
+        }
+    }
+
+
+
+    Op("Qr") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "Qr"
+
+        Input(DataType.NUMERIC, "input") {description = "input tensor"}
+        Arg(DataType.BOOL, "full") {description = "full matrices mode"; defaultValue = false}
+        Output(FLOATING_POINT, "outputQ")
+        Output(FLOATING_POINT, "outputR")
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Computes the QR decompositions of input matrix.
+            """.trimIndent()
+        }
+    }
+
+    Op("MatrixBandPart") {
+        javaPackage = "org.nd4j.linalg.api.ops.custom"
+        javaOpClass = "MatrixBandPart"
+
+        Input(DataType.NUMERIC, "input") { description = "input tensor" }
+        Arg(DataType.INT, "minLower") { description = "lower diagonal count" }
+        Arg(DataType.INT, "maxUpper") { description = "upper diagonal count" }
+        Output(DataType.FLOATING_POINT, "output1")
+        Output(DataType.FLOATING_POINT, "output2")
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Copy a tensor setting outside a central band in each innermost matrix.
+            """.trimIndent()
+        }
+    }
+
+    Op("cross") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.shape"
+        javaOpClass = "Cross"
+
+        Input(DataType.NUMERIC, "a") {"Input tensor a"}
+        Input(DataType.NUMERIC, "b") {"Input tensor b"}
+        Output(FLOATING_POINT, "output")
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Computes pairwise cross product.
+            """.trimIndent()
+        }
+    }
+
+    Op("diag") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.shape"
+        javaOpClass = "Diag"
+
+        Input(DataType.NUMERIC, "input") {"Input tensor"}
+        Output(DataType.FLOATING_POINT, "output")
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Calculates diagonal tensor.
+            """.trimIndent()
+        }
+    }
+
+    Op("diag_part") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.shape"
+        javaOpClass = "DiagPart"
+
+        Input(DataType.NUMERIC, "input") {"Input tensor"}
+        Output(DataType.FLOATING_POINT, "output")
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Calculates diagonal tensor.
+            """.trimIndent()
+        }
+    }
+
+
+
+
+    Op("matrixDeterminant") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "MatrixDeterminant"
+
+        Input(DataType.NUMERIC, "input") {"Input tensor"}
+        Output(FLOATING_POINT, "output")
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Calculates matrix determinant.
+            """.trimIndent()
+        }
+    }
+
+
+    Op("logdet") {
+        javaPackage = "org.nd4j.linalg.api.ops.custom"
+        javaOpClass = "Logdet"
+
+        Input(DataType.NUMERIC, "input") {"Input tensor"}
+        Output(FLOATING_POINT, "output")
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Calculates log of determinant.
+            """.trimIndent()
+        }
+    }
+
+    Op("matrixInverse") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "MatrixInverse"
+
+        Input(DataType.NUMERIC, "input") {"Input tensor"}
+        Output(FLOATING_POINT, "output")
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Inverts a matrix
+            """.trimIndent()
+        }
+    }
+
+
+    Op("eig") {
+        javaPackage = "org.nd4j.linalg.api.ops.custom"
+        javaOpClass = "Eig"
+
+        Input(DataType.NUMERIC, "input") {"Input tensor"}
+        Output(FLOATING_POINT, "eigenValues")
+        Output(FLOATING_POINT, "eigenVectors")
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Calculates eigen values
+            """.trimIndent()
+        }
+    }
+
+    Op("svd") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "Svd"
+
+        Input(DataType.NUMERIC, "input") {"Input tensor"}
+        Arg(DataType.BOOL, "fullUV") {"Full matrices mode"}
+        Arg(DataType.BOOL, "computeUV") {"Compute U and V"}
+        Arg(DataType.INT, "switchNum") {"Switch number"; defaultValue = 16}
+        Output(FLOATING_POINT, "output")
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Calculates singular value decomposition.
+            """.trimIndent()
+        }
+    }
+
+    Op("tri") {
+        javaPackage = "org.nd4j.linalg.api.ops.custom"
+        javaOpClass = "Tri"
+
+        Arg(DATA_TYPE, "dataType") { description = "Data type"; defaultValue = org.nd4j.linalg.api.buffer.DataType.FLOAT }
+        Arg(INT, "row") {"Number of rows in the array"; }
+        Arg(INT, "column") {"Number of columns in the array";  }
+        Arg(INT, "diagonal") {"The sub-diagonal at and below which the array is filled. k = 0 is the main diagonal, while k < 0 is below it, and k > 0 is above. The default is 0."; defaultValue =  0}
+
+
+        Output(FLOATING_POINT, "output")
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             An array with ones at and below the given diagonal and zeros elsewhere.
+            """.trimIndent()
+        }
+    }
+
+    Op("triu") {
+        javaPackage = "org.nd4j.linalg.api.ops.custom"
+        javaOpClass = "Triu"
+        Input(DataType.NUMERIC, "input") {"Input tensor"}
+        Arg(DataType.INT, "diag") {"diagonal"; defaultValue = 0}
+
+        Output(FLOATING_POINT, "output")
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Upper triangle of an array. Return a copy of a input tensor with the elements below the k-th diagonal zeroed.
+            """.trimIndent()
+        }
+    }
+    
+    Alias(SDBaseOps(), "mmul")
+}

codegen/op-codegen/src/main/ops/org/nd4j/codegen/ops/NeuralNetwork.kt

@@ -0,0 +1,602 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.ops
+
+import org.nd4j.codegen.api.AtLeast
+import org.nd4j.codegen.api.DataType.*
+import org.nd4j.codegen.api.Language
+import org.nd4j.codegen.api.doc.DocScope
+import org.nd4j.codegen.dsl.*
+import org.nd4j.codegen.mixins.transformStrict
+
+fun NN() = Namespace("NN") {
+    val convPkg = "org.nd4j.linalg.api.ops.impl.layers.convolution"
+
+    Op("batchNorm") {
+        javaPackage = convPkg
+        Input(NUMERIC, "input") { description = "Input variable." }
+        Input(NUMERIC, "mean") { description = "Mean value. For 1d axis, this should match input.size(axis)" }
+        Input(NUMERIC, "variance") { description = "Variance value. For 1d axis, this should match input.size(axis)" }
+        Input(NUMERIC, "gamma") { description = "Gamma value. For 1d axis, this should match input.size(axis)" }
+        Input(NUMERIC, "beta") { description = "Beta value. For 1d axis, this should match input.size(axis)" }
+        Arg(NUMERIC, "epsilon") { description = "Epsilon constant for numerical stability (to avoid division by 0)" }
+        Arg(INT, "axis") {
+            count = AtLeast(1)
+            description = "For 2d CNN activations: 1 for NCHW format activations, or 3 for NHWC format activations.\n" +
+                    "For 3d CNN activations: 1 for NCDHW format, 4 for NDHWC\n" +
+                    "For 1d/RNN activations: 1 for NCW format, 2 for NWC"
+        }
+
+        Output(NUMERIC, "output") { description = "variable for batch normalization" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+            Neural network batch normalization operation.
+            For details, see <a href="https://arxiv.org/abs/1502.03167">https://arxiv.org/abs/1502.03167</a>
+            """.trimIndent()
+        }
+    }
+
+    Op("biasAdd") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.broadcast"
+        Input(NUMERIC, "input") { description = "4d input variable" }
+        Input(NUMERIC, "bias") { description = "1d bias" }
+        Arg(BOOL, "nchw") { description = "The format - nchw=true means [minibatch, channels, height, width] format; nchw=false - [minibatch, height, width, channels].\n" +
+                "Unused for 2d inputs" }
+
+        Output(NUMERIC, "output") { description = "Output variable, after applying bias add operation" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+            Bias addition operation: a special case of addition, typically used with CNN 4D activations and a 1D bias vector
+            """.trimIndent()
+        }
+    }
+
+    Op("dropout") {
+        javaPackage = "org.nd4j.linalg.api.ops.random.impl"
+        javaOpClass = "CustomDropOut"
+        Input(NUMERIC, "input") { description = "Input array" }
+        Arg(BOOL, "inverted") { description = "Whether dropout should be inverted or not." }
+        Arg(INT, "seed") { description = "the seed for dropout"; defaultValue = 0 }
+        Arg(NUMERIC,"probabilityValue") { description = "the chance of dropping a value to 0. Maybe interpreted as 1 - p if inverted is true."}
+        Output(NUMERIC, "output") { description = "Output" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+                Dropout operation
+            """.trimIndent()
+        }
+    }
+
+
+
+    Op("elu", transformStrict) {
+        javaOpClass = "ELU"
+        legacy = false
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Element-wise exponential linear unit (ELU) function:
+             out = x if x > 0
+             out = a * (exp(x) - 1) if x <= 0
+             with constant a = 1.0
+             <p>
+             See: <a href="https://arxiv.org/abs/1511.07289">https://arxiv.org/abs/1511.07289</a>
+            """.trimIndent()
+        }
+    }
+
+    Op("gelu", transformStrict) {
+        javaOpClass = "GELU"
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             GELU activation function - Gaussian Error Linear Units
+             For more details, see <i>Gaussian Error Linear Units (GELUs)</i> - <a href="https://arxiv.org/abs/1606.08415">https://arxiv.org/abs/1606.08415</a>
+             This method uses the sigmoid approximation
+            """.trimIndent()
+        }
+    }
+
+    Op("hardSigmoid", transformStrict) {
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Element-wise hard sigmoid function:
+             out[i] = 0 if in[i] <= -2.5
+             out[1] = 0.2*in[i]+0.5 if -2.5 < in[i] < 2.5
+             out[i] = 1 if in[i] >= 2.5
+            """.trimIndent()
+        }
+    }
+
+    Op("hardTanh", transformStrict) {
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Element-wise hard tanh function:
+             out[i] = -1 if in[i] <= -1
+             out[1] = in[i] if -1 < in[i] < 1
+             out[i] = 1 if in[i] >= 1
+            """.trimIndent()
+        }
+    }
+
+    Op("hardTanhDerivative") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.gradient"
+        legacy = true
+        Input(NUMERIC, "x") { description = "Input variable" }
+        Output(NUMERIC, "output"){ description = "Output variable" }
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Derivative (dOut/dIn) of the element-wise hard Tanh function - hardTanh(%INPUT_TYPE%)
+            """.trimIndent()
+        }
+    }
+
+    Op("leakyRelu") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.scalar"
+        javaOpClass = "LeakyReLU"
+        legacy = true
+        Input(NUMERIC, "x") { description = "Input variable" }
+        Arg(NUMERIC, "alpha") { description = "Cutoff - commonly 0.01" }
+
+        Output(NUMERIC, "output") { description = "Output variable" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Element-wise leaky ReLU function:
+             out = x if x >= 0.0
+             out = alpha * x if x < cutoff
+             Alpha value is most commonly set to 0.01
+            """.trimIndent()
+        }
+    }
+
+    Op("leakyReluDerivative") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.gradient"
+        javaOpClass = "LeakyReLUDerivative"
+        legacy = true
+        Input(NUMERIC, "x") { description = "Input variable" }
+        Arg(FLOATING_POINT, "alpha") { description = "Cutoff - commonly 0.01" }
+
+        Output(NUMERIC, "output") { description = "Output variable" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Leaky ReLU derivative: dOut/dIn given input.
+            """.trimIndent()
+        }
+    }
+
+    Op("CReLU") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "CReLU"
+        Input(NUMERIC, "x") { description = "Input variable" }
+        Output(NUMERIC, "output") { description = "Output variable" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Concatenates a ReLU which selects only the positive part of the activation with a ReLU which selects only the negative part of the activation. Note that as a result this non-linearity doubles the depth of the activations.
+            """.trimIndent()
+        }
+    }
+
+    Op("linear") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "XwPlusB"
+        Input(NUMERIC, "input") { description = "Input data" }
+        Input(NUMERIC, "weights") { description = "Weights variable, shape [nIn, nOut]" }
+        Input(NUMERIC, "bias") { description = "Optional bias variable (may be null)" /*; optional = true*/ }
+        Arg(BOOL,"transposeA") { description = "Whether to transpose input or not"; defaultValue= false}
+        Arg(BOOL,"transposeB") { description = "Whether to transpose second input or not"; defaultValue= false}
+        Arg(BOOL,"transposeC") { description = "Whether to transpose result or not"; defaultValue= false}
+        Output(NUMERIC, "output") { description = "Output variable" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Linear layer operation: out = mmul(in,w) + bias
+             Note that bias array is optional
+            """.trimIndent()
+        }
+    }
+
+
+    Op("logSigmoid", transformStrict) {
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Element-wise sigmoid function: out[i] = log(sigmoid(in[i]))
+            """.trimIndent()
+        }
+    }
+
+    Op("logSoftmax") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "LogSoftMax"
+        Input(NUMERIC, "x") { description = "" }
+        Output(NUMERIC, "output") { description = "" }
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Log softmax activation
+            """.trimIndent()
+        }
+    }
+
+    Op("logSoftmax") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "LogSoftMax"
+        Input(NUMERIC, "x") { description = "Input" }
+        Arg(INT, "dimension") { description = "Dimension along which to apply log softmax" }
+        Output(NUMERIC, "output") { description = "Output - log(softmax(input))" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Log softmax activation
+            """.trimIndent()
+        }
+    }
+
+    Op("relu") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.scalar"
+        javaOpClass = "RectifiedLinear"
+        legacy = true
+        Input(NUMERIC, "x") { description = "Input" }
+        Arg(NUMERIC, "cutoff") { description = "Cutoff value for ReLU operation - x > cutoff ? x : 0. Usually 0" }
+        Output(NUMERIC, "output") { description = "Output" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Element-wise rectified linear function with specified cutoff:
+             out[i] = in[i] if in[i] >= cutoff
+             out[i] = 0 otherwise
+            """.trimIndent()
+        }
+    }
+
+    Op("relu6") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.scalar"
+        legacy = true
+        Input(NUMERIC, "x") { description = "Input" }
+        Arg(NUMERIC, "cutoff") { description = "Cutoff value for ReLU operation. Usually 0" }
+        Output(NUMERIC, "output") { description = "Output" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Element-wise "rectified linear 6" function with specified cutoff:
+             out[i] = min(max(in, cutoff), 6)
+            """.trimIndent()
+        }
+    }
+
+    Op("reluLayer") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms"
+        Input(NUMERIC, "input") { description = "Input data" }
+        Input(NUMERIC, "weights") { description = "Weights variable" }
+        Input(NUMERIC, "bias") { description = " Bias variable" }
+        Output(NUMERIC, "output") { description = "Output variable" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             ReLU (Rectified Linear Unit) layer operation: out = relu(mmul(in,w) + bias)
+           
+            """.trimIndent()
+        }
+    }
+
+    Op("preciseGelu", transformStrict) {
+        javaOpClass = "PreciseGELU"
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             GELU activation function - Gaussian Error Linear Units
+             For more details, see <i>Gaussian Error Linear Units (GELUs)</i> - <a href="https://arxiv.org/abs/1606.08415">https://arxiv.org/abs/1606.08415</a>
+             This method uses the precise method
+            """.trimIndent()
+        }
+    }
+
+    Op("prelu") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.scalar"
+        javaOpClass = "PRelu"
+        Input(NUMERIC, "input") { description = "Input data" }
+        Input(NUMERIC, "alpha") { description = "The cutoff variable.  Note that the batch dimension (the 0th, whether it is batch or not) should not be part of alpha." }
+        Arg(INT, "sharedAxes") { count = AtLeast(1); description = "Which axes to share cutoff parameters along." }
+
+        Output(NUMERIC, "output") { description = "Output" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             PReLU (Parameterized Rectified Linear Unit) operation.  Like LeakyReLU with a learnable alpha:
+             out[i] = in[i] if in[i] >= 0
+             out[i] = in[i] * alpha[i] otherwise
+            
+             sharedAxes allows you to share learnable parameters along axes.
+             For example, if the input has shape [batchSize, channels, height, width]
+             and you want each channel to have its own cutoff, use sharedAxes = [2, 3] and an
+             alpha with shape [channels].
+            """.trimIndent()
+        }
+    }
+
+    Op("selu", transformStrict) {
+        javaOpClass = "SELU"
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Element-wise SeLU function - Scaled exponential Lineal Unit: see <a href="https://arxiv.org/abs/1706.02515">Self-Normalizing Neural Networks</a>
+             
+             out[i] = scale * alpha * (exp(in[i])-1) if in[i]>0, or 0 if in[i] <= 0
+             Uses default scale and alpha values.
+            """.trimIndent()
+        }
+    }
+
+    Op("sigmoid", transformStrict) {
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Element-wise sigmoid function: out[i] = 1.0/(1+exp(-in[i]))
+            """.trimIndent()
+        }
+    }
+
+    Op("sigmoidDerivative") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.gradient"
+        Input(NUMERIC, "x") { description = "Input Variable" }
+        Input(NUMERIC, "wrt") { description = "Gradient at the output - dL/dOut. Must have same shape as the input" }
+        Output(NUMERIC, "output") { description = "Output (gradient at input of sigmoid)" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Element-wise sigmoid function derivative: dL/dIn given input and dL/dOut
+            """.trimIndent()
+        }
+    }
+
+    Op("softmax") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        javaOpClass = "SoftMax"
+        Input(NUMERIC, "x") { description = "Input" }
+        Arg(INT, "dimension") { description = "Dimension along which to apply softmax"; defaultValue = -1 }
+        Output(NUMERIC, "output") { description = "Output variable" }
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Softmax activation, along the specified dimension
+            """.trimIndent()
+        }
+    }
+
+
+
+    Op("softplus", transformStrict) {
+        javaOpClass = "SoftPlus"
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Element-wise softplus function: out = log(exp(x) + 1)
+            """.trimIndent()
+        }
+    }
+
+    Op("softsign", transformStrict) {
+        javaOpClass = "SoftSign"
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Element-wise softsign function: out = x / (abs(x) + 1)
+            """.trimIndent()
+        }
+    }
+
+    Op("softsignDerivative") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.gradient"
+        javaOpClass = "SoftSignDerivative"
+        legacy = true
+        Input(NUMERIC, "x") { description = "Input variable" }
+        Output(NUMERIC, "output") { description = "Output" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Element-wise derivative (dOut/dIn) of the softsign function softsign(%INPUT_TYPE%)
+            """.trimIndent()
+        }
+    }
+
+    Op("swish", transformStrict) {
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Element-wise "swish" function: out = x * sigmoid(b*x) with b=1.0
+             See: <a href="https://arxiv.org/abs/1710.05941">https://arxiv.org/abs/1710.05941</a>
+            """.trimIndent()
+        }
+    }
+
+    Op("layerNorm") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        val input = Input(NUMERIC, "input") { description = "Input variable" }
+        val g = Input(NUMERIC, "gain") { description = "Gain" }
+        Input(NUMERIC, "bias") { description = "Bias"; defaultValue = null}
+        val ch = Arg(BOOL, "channelsFirst") { description = "For 2D input - unused. True for NCHW (minibatch, channels, height, width), false for NHWC data" }
+        val dim = Arg(LONG, "dimensions") { count = AtLeast(1); description = "Dimensions to perform layer norm over - dimension=1 for 2d/MLP data, dimension=1,2,3 for CNNs" }
+
+        Output(NUMERIC, "output") { description = "Output variable" }
+
+        AllParamSignature()
+        Signature(input, g, ch, dim)
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Apply Layer Normalization
+            
+             y = gain * standardize(x) + bias
+            """.trimIndent()
+        }
+    }
+
+
+    Op("dotProductAttentionV2") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        val q = Input(NUMERIC, "queries") { description = "A {@link SDVariable} representing the query tensor. Shape: [batchSize, numQueries, queryDim]" }
+        val v = Input(NUMERIC, "values") { description = "A {@link SDVariable} representing the value tensor. Shape: [batchSize, numValues, valueDim]" }
+
+        val k = Input(NUMERIC, "keys") { description = "A {@link SDVariable} representing the key tensor. Shape: [batchSize, numValues, keyDim]" }
+        val queryMask = Input(NUMERIC, "queryMask") { description = "A {@link SDVariable} representing the query mask tensor. Shape: [batchSize, numQueries]" }
+        val valueMask = Input(NUMERIC, "valueMask") { description = "@param valueMask          A {@link SDVariable} representing the value mask tensor. Shape: [batchSize, numValues]" }
+
+        val s = Arg(FLOATING_POINT, "scaleFactor") { defaultValue = 1.0; description = "@param scaleFactor        A {@code double} scaling factor applied to the dot product between queries and keys." }
+        val dropout = Arg(FLOATING_POINT, "dropoutProbability") { defaultValue = 0.0; description = "A {@code double} specifying the dropout probability to be applied to attention weights." }
+        val useCausalMask = Arg(BOOL, "useCausalMask") { defaultValue = false; description = " A {@code boolean} flag to indicate whether to apply a causal mask to the attention scores, for autoregressive tasks." }
+        val training = Arg(BOOL, "training") { defaultValue = false; description = " A {@code boolean} flag to indicate whether the layer is in training mode or inference mode, affecting dropout." }
+
+        Output(NUMERIC, "output") { description = " A {@link SDVariable} representing the output tensor of the dot product attention operation. Shape: [batchSize, numQueries, valueDim]"}
+
+        Signature(q,v,k,queryMask,valueMask, s,dropout,useCausalMask,training)
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             This operation performs dot product attention on the given timeseries input with the given queries
+             out = sum(similarity(k_i, q) * v_i)
+            
+             similarity(k, q) = softmax(k * q) where x * q is the dot product of x and q
+            
+             Optionally with normalization step:
+             similarity(k, q) = softmax(k * q / sqrt(size(q))
+            
+             See also "Attention is all you need" (https://arxiv.org/abs/1706.03762, p. 4, eq. 1)
+            
+             Note: This supports multiple queries at once, if only one query is available the queries vector still has to
+             be 3D but can have queryCount = 1
+            
+             Note: keys and values usually is the same array. If you want to use it as the same array, simply pass it for
+             both.
+            
+             Note: Queries, keys and values must either be all rank 3 or all rank 4 arrays. Mixing them doesn't work. The
+             output rank will depend on the input rank.
+            """.trimIndent()
+        }
+    }
+
+    Op("dotProductAttention") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        val q = Input(NUMERIC, "queries") { description = "input 3D array \"queries\" of shape [batchSize, featureKeys, queryCount]\n" +
+                "or 4D array of shape [batchSize, numHeads, featureKeys, queryCount]" }
+        val k = Input(NUMERIC, "keys") { description = "input 3D array \"keys\" of shape [batchSize, featureKeys, timesteps]\n" +
+                "or 4D array of shape [batchSize, numHeads, featureKeys, timesteps]" }
+        val v = Input(NUMERIC, "values") { description = "input 3D array \"values\" of shape [batchSize, featureValues, timesteps]\n" +
+                "or 4D array of shape [batchSize, numHeads, featureValues, timesteps]" }
+        val m = Input(NUMERIC, "mask") { description = "OPTIONAL; array that defines which values should be skipped of shape [batchSize, timesteps]" }
+        val s = Arg(BOOL, "scaled") { description = "normalization, false -> do not apply normalization, true -> apply normalization" }
+        Arg(BOOL, "withWeights") { defaultValue = false; description = "withWeights return attention weights as well, false -> only one output, true -> two outputs" }
+
+        Output(NUMERIC, "output") { description = " Attention result arrays of shape [batchSize, featureValues, queryCount] or [batchSize, numHeads, featureValues, queryCount],\n" +
+                "(optionally) Attention Weights of shape [batchSize, timesteps, queryCount] or [batchSize, numHeads, timesteps, queryCount]" }
+
+        Signature(q, k, v, m, s)
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             This operation performs dot product attention on the given timeseries input with the given queries
+             out = sum(similarity(k_i, q) * v_i)
+            
+             similarity(k, q) = softmax(k * q) where x * q is the dot product of x and q
+            
+             Optionally with normalization step:
+             similarity(k, q) = softmax(k * q / sqrt(size(q))
+            
+             See also "Attention is all you need" (https://arxiv.org/abs/1706.03762, p. 4, eq. 1)
+            
+             Note: This supports multiple queries at once, if only one query is available the queries vector still has to
+             be 3D but can have queryCount = 1
+            
+             Note: keys and values usually is the same array. If you want to use it as the same array, simply pass it for
+             both.
+            
+             Note: Queries, keys and values must either be all rank 3 or all rank 4 arrays. Mixing them doesn't work. The
+             output rank will depend on the input rank.
+            """.trimIndent()
+        }
+    }
+
+
+
+    Op("multiHeadDotProductAttention") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        val q = Input(NUMERIC, "queries") { description = "input 3D array \"queries\" of shape [batchSize, featureKeys, queryCount]" }
+        val k = Input(NUMERIC, "keys") { description = "input 3D array \"keys\" of shape [batchSize, featureKeys, timesteps]" }
+        val v = Input(NUMERIC, "values") { description = "input 3D array \"values\" of shape [batchSize, featureValues, timesteps]" }
+        val wq = Input(NUMERIC, "Wq") { description = "input query projection weights of shape [numHeads, projectedKeys, featureKeys]" }
+        val wk = Input(NUMERIC, "Wk") { description = "input key projection weights of shape [numHeads, projectedKeys, featureKeys]" }
+        val wv = Input(NUMERIC, "Wv") { description = "input value projection weights of shape [numHeads, projectedValues, featureValues]" }
+        val wo = Input(NUMERIC, "Wo") { description = "output projection weights of shape [numHeads * projectedValues, outSize]" }
+        val m = Input(NUMERIC, "mask") { description = "OPTIONAL; array that defines which values should be skipped of shape [batchSize, timesteps]" }
+        val s = Arg(BOOL, "scaled") { description = "normalization, false -> do not apply normalization, true -> apply normalization" }
+        Arg(BOOL, "withWeights") { defaultValue = false; description = "return attention weights as well, false -> only one output, true -> two outputs" }
+
+        Output(NUMERIC, "output") { description = "Attention result arrays of shape [batchSize, outSize, queryCount]\n" +
+                "(optionally) Attention Weights of shape [batchSize, numHeads, timesteps, queryCount]" }
+
+        Signature(q, k, v, wq, wk, wv, wo, m, s)
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             This performs multi-headed dot product attention on the given timeseries input
+             out = concat(head_1, head_2, ..., head_n) * Wo
+             head_i = dot_product_attention(Wq_i*q, Wk_i*k, Wv_i*v)
+            
+             Optionally with normalization when calculating the attention for each head.
+            
+             See also "Attention is all you need" (https://arxiv.org/abs/1706.03762, pp. 4,5, "3.2.2 Multi-Head Attention")
+            
+             This makes use of dot_product_attention OP support for rank 4 inputs.
+             see dotProductAttention(%INPUT_TYPE%, %INPUT_TYPE%, %INPUT_TYPE%, %INPUT_TYPE%, boolean, boolean)
+            """.trimIndent()
+        }
+    }
+
+    Op("pad") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms"
+        Input(NUMERIC, "input") { description = "Input tensor"}
+        Input(NUMERIC, "padding") { description = "Padding value" }
+        Arg(ENUM, "PadMode") { possibleValues = listOf("CONSTANT", "REFLECT", "SYMMETRIC"); description = "Padding format"; defaultValue="CONSTANT" }
+        Arg(NUMERIC, "constant") { description = "Padding constant" }
+
+        Output(NUMERIC, "output"){ description = "Padded input" }
+
+        Doc(Language.ANY, DocScope.ALL){
+            """
+             Padding operation 
+            """.trimIndent()
+        }
+    }
+
+
+
+    Op("topK") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
+        Input(NUMERIC, "input") { description = "Input data" }
+        Arg(NUMERIC, "k") { description = "The number of values to return" }
+        Arg(BOOL, "sorted") { description = "Whether to return the values sorted or not" }
+        Output(NUMERIC, "output") { description = "the top k values in the input" }
+        Output(NUMERIC, "indices") { description = "the indices of the top k values" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Find values and indices for the largest k entries along the last dimension.<br>
+            """.trimIndent()
+        }
+    }
+
+    Alias(Math(), "tanh")
+}

codegen/op-codegen/src/main/ops/org/nd4j/codegen/ops/RNN.kt

@@ -0,0 +1,356 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.ops
+
+import org.nd4j.codegen.api.DataType.*
+import org.nd4j.codegen.api.Language
+import org.nd4j.codegen.api.doc.DocScope
+import org.nd4j.codegen.dsl.*
+
+fun SDRNN() = Namespace("RNN") {
+
+
+    val LSTMConfiguration = Config("LSTMConfiguration") {
+
+        Arg(ENUM, "RnnDataFormat") {
+            possibleValues = listOf("TNS", "NST", "NTS"); description = " The data format of the input. Input shape depends on data format (in config):<br>\n" +
+                " TNS -> [timeSteps, batchSize, inSize]<br>\n" +
+                " NST -> [batchSize, inSize, timeSteps]<br>\n" +
+                " NTS -> [batchSize, timeSteps, inSize]<br>"
+        }
+
+
+        Arg(BOOL, "peepHole") { description = "Whether to provide peephole connections"; }
+        Arg(NUMERIC, "forgetBias") { description = "The bias added to forget gates in order to reduce the scale of forgetting in the beginning of the training."; }
+        Arg(NUMERIC, "clippingCellValue") { description = "The bias added to forget gates in order to reduce the scale of forgetting in the beginning of the training."; }
+
+        javaClassOverride = "org.nd4j.linalg.api.ops.impl.layers.recurrent.config.LSTMConfiguration"
+    }
+
+
+    val LSTMLayerConfig = Config("LSTMLayerConfig") {
+
+        Arg(ENUM, "LSTMDataFormat") {
+            possibleValues = listOf("TNS", "NST", "NTS", "T2NS");
+            description = "for unidirectional:" +
+                    "  TNS: shape [timeLength, numExamples, inOutSize] - sometimes referred to as \"time major\"<br>\n" +
+                    "  NST: shape [numExamples, inOutSize, timeLength]<br>\n" +
+                    "  NTS: shape [numExamples, timeLength, inOutSize] - TF \"time_major=false\" layout<br>" +
+                    " for bidirectional:\n" +
+                    "   T2NS: 3 = [timeLength, 2, numExamples, inOutSize] (for ONNX)"
+        }
+
+
+        Arg(ENUM, "LSTMDirectionMode") {
+            possibleValues = listOf("FWD", "BWD", "BIDIR_SUM", "BIDIR_CONCAT", "BIDIR_EXTRA_DIM"); description = "direction <br>\n" +
+                " FWD: 0 = fwd\n" +
+                " BWD: 1 = bwd\n" +
+                " BIDIR_SUM: 2 = bidirectional sum\n" +
+                " BIDIR_CONCAT: 3 = bidirectional concat\n" +
+                " BIDIR_EXTRA_DIM: 4 = bidirectional extra output dim (in conjunction with format dataFormat = 3)"
+        }
+
+        Arg(ENUM, "gateAct") {
+            possibleValues = listOf("TANH",
+                    "RELU",
+                    "SIGMOID",
+                    "AFFINE",
+                    "LEAKY_RELU",
+                    "THRESHHOLD_RELU",
+                    "SCALED_TAHN",
+                    "HARD_SIGMOID",
+                    "ELU",
+                    "SOFTSIGN",
+                    "SOFTPLUS"); description = "Activations"
+        }
+
+
+        Arg(ENUM, "cellAct") {
+            possibleValues = listOf("TANH",
+                    "RELU",
+                    "SIGMOID",
+                    "AFFINE",
+                    "LEAKY_RELU",
+                    "THRESHHOLD_RELU",
+                    "SCALED_TAHN",
+                    "HARD_SIGMOID",
+                    "ELU",
+                    "SOFTSIGN",
+                    "SOFTPLUS"); description = "Activations"
+        }
+
+
+        Arg(ENUM, "outAct") {
+            possibleValues = listOf("TANH",
+                    "RELU",
+                    "SIGMOID",
+                    "AFFINE",
+                    "LEAKY_RELU",
+                    "THRESHHOLD_RELU",
+                    "SCALED_TAHN",
+                    "HARD_SIGMOID",
+                    "ELU",
+                    "SOFTSIGN",
+                    "SOFTPLUS"); description = "Activations"
+        }
+
+
+        Arg(BOOL, "retFullSequence") { description = "indicates whether to return whole time sequence h {h_0, h_1, ... , h_sL-1}"; defaultValue = true }
+        Arg(BOOL, "retLastH") {
+            description = "indicates whether to return output at last time step only,\n" +
+                    " in this case shape would be [bS, nOut] (exact shape depends on dataFormat argument)"; defaultValue = false
+        }
+        Arg(BOOL, "retLastC") {
+            description = "indicates whether to return cells state at last time step only,\n" +
+                    " in this case shape would be [bS, nOut] (exact shape depends on dataFormat argument)"; defaultValue = false
+        }
+        Arg(NUMERIC, "cellClip") { description = "Cell clipping value, if it = 0 then do not apply clipping"; defaultValue = 0.0}
+
+        Arg(NUMERIC, "gateAlpha") {defaultValue=0.0}
+        Arg(NUMERIC, "gateBeta") {defaultValue=0.0}
+        Arg(NUMERIC, "cellAlpha") {defaultValue=0.0}
+        Arg(NUMERIC, "cellBeta") {defaultValue=0.0}
+        Arg(NUMERIC, "outAlpha") {defaultValue=0.0}
+        Arg(NUMERIC, "outBeta") {defaultValue=0.0}
+
+
+       javaClassOverride =  "org.nd4j.linalg.api.ops.impl.layers.recurrent.config.LSTMLayerConfig"
+    }
+
+
+    val GRUWeights = Config("GRUWeights") {
+        Input(NUMERIC, "ruWeight")
+        Input(NUMERIC, "cWeight")
+        Input(NUMERIC, "ruBias")
+        Input(NUMERIC, "cBias")
+        javaClassOverride = "org.nd4j.linalg.api.ops.impl.layers.recurrent.weights.GRUWeights"
+    }
+
+    val SRUWeights = Config("SRUWeights") {
+        Input(NUMERIC, "weights")
+        Input(NUMERIC, "bias")
+        javaClassOverride = "org.nd4j.linalg.api.ops.impl.layers.recurrent.weights.SRUWeights"
+    }
+
+    val LSTMWeights = Config("LSTMWeights") {
+        Input(NUMERIC, "ruWeight")
+        Input(NUMERIC, "inputPeepholeWeights")
+        Input(NUMERIC, "forgetPeepholeWeights")
+        Input(NUMERIC, "outputPeepholeWeights")
+        Input(NUMERIC, "bias")
+
+        javaClassOverride = "org.nd4j.linalg.api.ops.impl.layers.recurrent.weights.LSTMWeights"
+    }
+
+    val LSTMLayerWeights = Config("LSTMLayerWeights") {
+        Input(NUMERIC, "inputWeights") {description="input weights Wx:\n" +
+                " 1) shapes `[nIn, 4*nOut]` for FWD,BWD " +
+                " 2) shapes `[2, nIn, 4*nOut]` BIDIR_SUM, BIDIR_CONCAT and BIDIR_EXTRA_DIM"}
+        Input(NUMERIC, "recurrentWeights") {description="recurrent weights Wr:\n" +
+                " 1) shapes `[nIn, 4*nOut]` for FWD, BWD " +
+                " 2) shapes `[2, nIn, 4*nOut]` BIDIR_SUM, BIDIR_CONCAT and BIDIR_EXTRA_DIM"}
+        Input(NUMERIC, "biases") {description="biases\n"+
+                " 1) shapes `[4*nOut]` for FWD, BWD " +
+                " 2) shapes `[2, 4*nOut]` for BIDIR_SUM, BIDIR_CONCAT and BIDIR_EXTRA_DIM"
+                  defaultValue=null}
+        Input(NUMERIC, "peepholeWeights") {description="peephole weights Wp:\n" +
+                "  1) `[3*nOut]`    when directionMode <  2\n" +
+                "  2) `[2, 3*nOut]`  when directionMode >= 2"; defaultValue=null}
+
+
+        javaClassOverride = "org.nd4j.linalg.api.ops.impl.layers.recurrent.weights.LSTMLayerWeights"
+    }
+
+
+    val namespaceJavaPackage = "org.nd4j.linalg.api.ops.impl.layers.recurrent"
+    Op("gruCell") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "GRUCell"
+        Input(NUMERIC, "x") { description = "Input, with shape [batchSize, inSize]" }
+        Input(NUMERIC, "hLast") { description = "Output of the previous cell/time step, with shape [batchSize, numUnits]" }
+        useConfig(GRUWeights)
+        Output(NUMERIC, "r") { description = "Reset gate output" }
+        Output(NUMERIC, "u") { description = "Update gate output" }
+        Output(NUMERIC, "c") { description = "Cell gate output" }
+        Output(NUMERIC, "h") { description = "Cell output" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+            The GRU cell.  Does a single time step operation
+            """.trimIndent()
+        }
+    }
+
+    Op("gru") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "GRU"
+        Input(NUMERIC, "x") { description = "input [time, bS, nIn]" }
+        Input(NUMERIC, "hLast") { description = "initial cell output (at time step = 0) [bS, nOut]" }
+        Input(NUMERIC, "Wx") { description = "input-to-hidden  weights, [nIn, 3*nOut]" }
+        Input(NUMERIC, "Wh") { description = "hidden-to-hidden weights, [nOut, 3*nOut]" }
+        Input(NUMERIC, "biases") { description = "biases, [3*nOut]" }
+
+        Output(NUMERIC, "h") { description = "cell outputs [time, bS, nOut], that is per each time step" }
+
+
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+            The GRU operation. Gated Recurrent Unit - Cho et al. 2014.
+
+
+            """.trimIndent()
+        }
+    }
+
+
+
+
+
+
+    Op("lstmCell") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "LSTMBlockCell"
+        Input(NUMERIC, "x") { description = "Input, with shape [batchSize, inSize]" }
+        Input(NUMERIC, "cLast") { description = "Previous cell state, with shape [batchSize, numUnits]" }
+        Input(NUMERIC, "yLast") { description = "revious cell output, with shape [batchSize, numUnits]" }
+        useConfig(LSTMWeights)
+        useConfig(LSTMConfiguration)
+
+        Output(NUMERIC, "i") { description = "Output - input modulation gate activations [batchSize, numUnits]." }
+        Output(NUMERIC, "c") { description = "Output - Activations, cell state (pre tanh) [batchSize, numUnits]." }
+        Output(NUMERIC, "f") { description = "Output - forget gate activations [batchSize, numUnits]." }
+        Output(NUMERIC, "o") { description = "Output - output gate activations [batchSize, numUnits]." }
+        Output(NUMERIC, "z") { description = "Output - input gate activations [batchSize, numUnits]." }
+        Output(NUMERIC, "h") { description = "Cell state, post tanh [batchSize, numUnits]." }
+        Output(NUMERIC, "y") { description = "Current cell output [batchSize, numUnits]." }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+            The LSTM cell.  Does a single time step operation.
+            """.trimIndent()
+        }
+    }
+
+
+
+    Op("lstmblock") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "LSTMBlock"
+        Input(NUMERIC, "maxTSLength") {defaultValue=null}
+        Input(NUMERIC, "x") { description = " Input, with shape dependent on the data format (in config)." }
+        Input(NUMERIC, "cLast") { description = "Previous/initial cell state, with shape [batchSize, numUnits]" ; defaultValue=null}
+        Input(NUMERIC, "yLast") { description = "Previous/initial cell output, with shape [batchSize, numUnits]" ; defaultValue=null }
+        useConfig(LSTMWeights)
+        useConfig(LSTMConfiguration)
+
+        Output(NUMERIC, "output") { description = "The layer's outputs." }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             The LSTM block
+            """.trimIndent()
+        }
+    }
+
+
+
+    Op("lstmLayer") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "LSTMLayer"
+        Input(NUMERIC, "x") { description = " Input, with shape dependent on the data format (in config)." }
+        Input(NUMERIC, "cLast") { description = "Previous/initial cell state, with shape [batchSize, numUnits]"; defaultValue=null }
+        Input(NUMERIC, "yLast") { description = "Previous/initial cell output, with shape [batchSize, numUnits]"; defaultValue=null }
+        Input(NUMERIC, "maxTSLength") { description = "maxTSLength with shape [batchSize]"; defaultValue=null }
+        useConfig(LSTMLayerWeights)
+        useConfig(LSTMLayerConfig)
+
+        //TODO these are optional
+        Output(NUMERIC, "output") { description = "The layer's outputs - full time series" }
+        Output(NUMERIC, "yLast") { description = "The layer's outputs - last time step activations (yLast)" }
+        Output(NUMERIC, "cLast") { description = "The layer's outputs - last time step cell state (cLast)" }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             Long Short-Term Memory layer - Hochreiter 1997.
+             SUPPORTS following data formats:
+             for unidirectional:
+             TNS: shapes [timeLength, numExamples, inOutSize]
+             NST: shapes [numExamples, inOutSize, timeLength]
+             NTS: shapes [numExamples, timeLength, inOutSize]
+             for bidirectional:
+             T2NS: shapes [timeLength, 2, numExamples, inOutSize] (for ONNX)
+             SUPPORTS following direction modes:
+             FWD: forward
+             BWD: backward
+             BIDIR_SUM: bidirectional sum
+             BIDIR_CONCAT: bidirectional concat
+             BIDIR_EXTRA_DIM: bidirectional extra output dim (in conjunction with format dataFormat - T2NS)
+             You may use different gate configurations:
+             specify gate/cell/out aplha/beta and numbers of activations for gate/cell/out described in activations enum
+             ("RELU","SIGMOID","AFFINE","LEAKY_RELU","THRESHHOLD_RELU","SCALED_TAHN","HARD_SIGMOID","ELU","SOFTSIGN","SOFTPLUS")
+             Also this layer supports MKLDNN (DNNL) and cuDNN acceleration
+            """.trimIndent()
+        }
+    }
+
+
+
+    Op("sruCell") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "SRUCell"
+        Input(NUMERIC, "x") { description = "Input, with shape [batchSize, inSize]" }
+        Input(NUMERIC, "cLast") { description = "Previous cell state, with shape [batchSize, inSize]" }
+        useConfig(SRUWeights)
+
+        Output(NUMERIC, "output") { description = "The cell's outputs." }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             The SRU layer.  Does a single time step operation.
+            """.trimIndent()
+        }
+    }
+
+
+    Op("sru") {
+        javaPackage = namespaceJavaPackage
+        javaOpClass = "SRU"
+        Input(NUMERIC, "x") { description = "Input, with shape [batchSize, inSize]" }
+        Input(NUMERIC, "initialC") { description = "Initial cell state, with shape [batchSize, inSize]" }
+        Input(NUMERIC, "mask") { description = "An optional dropout mask, with shape [batchSize, inSize]"; defaultValue = null }
+
+        useConfig(SRUWeights)
+
+        Output(NUMERIC, "output") { description = "The cell's outputs.." }
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+             The SRU layer.  Does a single time step operation.
+            """.trimIndent()
+        }
+
+    }
+}
+
+
+
+

codegen/op-codegen/src/main/ops/org/nd4j/codegen/ops/Random.kt

@@ -0,0 +1,142 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.ops
+
+import org.nd4j.codegen.api.AtLeast
+import org.nd4j.codegen.api.DataType.*
+import org.nd4j.codegen.api.Language
+import org.nd4j.codegen.api.doc.DocScope
+import org.nd4j.codegen.dsl.*
+
+fun Random() = Namespace("Random") {
+    val random = Mixin("random"){
+        Arg(DATA_TYPE, "datatype"){ description = "Data type of the output variable"}
+        Arg(LONG, "shape") { count = AtLeast(0); description = "Shape of the new random %INPUT_TYPE%, as a 1D array" }
+        Output(NUMERIC, "output") { description = "Tensor with the given shape where values are randomly sampled according to a %OP_NAME% distribution" }
+    }
+
+    val legacyRandom = Mixin("legacyRandom"){
+        useMixin(random)
+        javaPackage = "org.nd4j.linalg.api.ops.random.impl"
+        legacy = true
+    }
+
+    val normalRandom = Mixin("normalRandom"){
+        Arg(FLOATING_POINT, "mean") { description = "Mean value for the random array" }
+        Arg(FLOATING_POINT, "stddev") { description = "Standard deviation for the random array" }
+        useMixin(legacyRandom)
+    }
+
+    Op("bernoulli") {
+        javaOpClass = "BernoulliDistribution"
+        Arg(FLOATING_POINT, "p") { description = "Probability of value 1" }
+        useMixin(legacyRandom)
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+            Generate a new random %INPUT_TYPE%, where values are randomly sampled according to a Bernoulli distribution,
+            with the specified probability. Array values will have value 1 with probability P and value 0 with probability
+            1-P.
+            """.trimIndent()
+        }
+    }
+
+    Op("binomial") {
+        javaOpClass = "BinomialDistribution"
+
+        Arg(INT, "nTrials") { description = "Number of trials parameter for the binomial distribution" }
+        Arg(FLOATING_POINT, "p") { description = "Probability of success for each trial" }
+        useMixin(legacyRandom)
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+            Generate a new random %INPUT_TYPE%, where values are randomly sampled according to a Binomial distribution,
+            with the specified number of trials and probability.
+            """.trimIndent()
+        }
+    }
+
+    Op("exponential") {
+        javaPackage = "org.nd4j.linalg.api.ops.random.custom"
+        javaOpClass = "RandomExponential"
+
+        val lambda = Arg(FLOATING_POINT, "lambda") { description = "lambda parameter" }
+        Constraint("Must be positive") { lambda gt 0 }
+        useMixin(random)
+
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+            Generate a new random %INPUT_TYPE%, where values are randomly sampled according to a exponential distribution:
+            P(x) = lambda * exp(-lambda * x)
+            """.trimIndent()
+        }
+    }
+
+    Op("logNormal", normalRandom) {
+        javaOpClass = "LogNormalDistribution"
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+            Generate a new random %INPUT_TYPE%, where values are randomly sampled according to a Log Normal distribution,
+            i.e., {@code log(x) ~ N(mean, stdev)}
+            """.trimIndent()
+        }
+    }
+
+    Op("normal", normalRandom) {
+        javaPackage = "org.nd4j.linalg.api.ops.random.impl"
+        javaOpClass = "GaussianDistribution"
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+            Generate a new random %INPUT_TYPE%, where values are randomly sampled according to a Gaussian (normal) distribution,
+            N(mean, stdev)<br>
+            """.trimIndent()
+        }
+    }
+
+    Op("normalTruncated", normalRandom) {
+        javaOpClass = "TruncatedNormalDistribution"
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+            Generate a new random %INPUT_TYPE%, where values are randomly sampled according to a Gaussian (normal) distribution,
+            N(mean, stdev). However, any values more than 1 standard deviation from the mean are dropped and re-sampled
+            """.trimIndent()
+        }
+    }
+
+    Op("uniform") {
+        javaOpClass = "UniformDistribution"
+
+        Arg(FLOATING_POINT, "min") { description = "Minimum value" }
+        Arg(FLOATING_POINT, "max") { description = "Maximum value." }
+        useMixin(legacyRandom)
+
+        Doc(Language.ANY, DocScope.ALL) {
+            """
+            Generate a new random %INPUT_TYPE%, where values are randomly sampled according to a uniform distribution,
+            U(min,max)
+            """.trimIndent()
+        }
+    }
+}

codegen/op-codegen/src/main/ops/org/nd4j/codegen/ops/SDLoss.kt

@@ -0,0 +1,278 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+/**
+ * Generated using ExtractFromExisting.kt
+ */
+
+package org.nd4j.codegen.ops
+
+import org.nd4j.codegen.api.Language
+import org.nd4j.codegen.api.doc.DocScope
+import org.nd4j.codegen.dsl.*
+import org.nd4j.codegen.api.DataType.*
+import org.nd4j.codegen.api.LossReduce
+
+fun SDLoss() =  Namespace("Loss"){
+
+    Op("absoluteDifference") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.loss"
+        javaOpClass = "AbsoluteDifferenceLoss"
+        Input(NUMERIC, "label") { description = "Label array" }
+        Input(NUMERIC, "predictions") { description = "Predictions array" }
+        Input(NUMERIC, "weights") { description = "Weights array. May be null. If null, a weight of 1.0 is used" }
+        Arg(LOSS_REDUCE, "lossReduce") { description = "Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT"; defaultValue = LossReduce.MEAN_BY_NONZERO_WEIGHT_COUNT}
+        Output(NUMERIC, "output"){ description = "loss variable" }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                Absolute difference loss: {@code sum_i abs( label[i] - predictions[i] )}
+            """.trimIndent()
+        }
+    }
+
+
+    Op("ctcLoss") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.loss"
+        javaOpClass = "CtcLoss"
+        Input(NUMERIC, "targetLabels") { description = "Label array" }
+        Input(NUMERIC, "logitInput") { description = "Inputs" }
+        Input(NUMERIC, "targetLabelLengths") { description = "Length of the target label" }
+        Input(NUMERIC, "logitInputLengths") { description = "Length of the input"}
+        Arg(INT,"blankIndex") {description = "The index of the blank label"; defaultValue = 0}
+        Output(NUMERIC, "output"){ description = "Ctc loss " }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                CTC Loss: Connectionist Temporal Classification Loss. See:
+                https://dl.acm.org/citation.cfm?id=1143891
+            """.trimIndent()
+        }
+    }
+
+    Op("cosineDistance") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.loss"
+        javaOpClass = "CosineDistanceLoss"
+        Input(NUMERIC, "label") { description = "Label array" }
+        Input(NUMERIC, "predictions") { description = "Predictions array" }
+        Input(NUMERIC, "weights") { description = "Weights array. May be null. If null, a weight of 1.0 is use" }
+        Arg(LOSS_REDUCE, "lossReduce") { description = "Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT"; defaultValue = LossReduce.MEAN_BY_NONZERO_WEIGHT_COUNT}
+        Arg(INT, "dimension") { description = "Dimension to perform the cosine distance over" }
+        Output(NUMERIC, "output"){ description = "Cosine distance loss " }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                Cosine distance loss: {@code 1 - cosineSimilarity(x,y)} or {@code 1 - sum_i label[i] * prediction[i]}, which is
+                equivalent to cosine distance when both the predictions and labels are normalized.<br>
+                <b>Note</b>: This loss function assumes that both the predictions and labels are normalized to have unit l2 norm.
+                If this is not the case, you should normalize them first by dividing by norm2(String, SDVariable, boolean, int...)
+                along the cosine distance dimension (with keepDims=true).
+            """.trimIndent()
+        }
+    }
+
+    Op("hingeLoss") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.loss"
+        javaOpClass = "HingeLoss"
+        Input(NUMERIC, "label") { description = "Label array. Each value should be 0.0 or 1.0 (internally -1 to 1 is used)" }
+        Input(NUMERIC, "predictions") { description = "Predictions array" }
+        Input(NUMERIC, "weights") { description = "Weights array. May be null. If null, a weight of 1.0 is used" }
+        Arg(LOSS_REDUCE, "lossReduce") { description = "Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT"; defaultValue = LossReduce.MEAN_BY_NONZERO_WEIGHT_COUNT}
+        Output(NUMERIC, "output"){ description = "Loss variable" }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                Hinge loss: a loss function used for training classifiers.
+                Implements {@code L = max(0, 1 - t * predictions)} where t is the label values after internally converting to {-1,1}
+                from the user specified {0,1}. Note that Labels should be provided with values {0,1}.
+            """.trimIndent()
+        }
+    }
+
+
+    Op("huberLoss") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.loss"
+        javaOpClass = "HuberLoss"
+        Input(NUMERIC, "label") { description = "Label array" }
+        Input(NUMERIC, "predictions") { description = "Predictions array" }
+        Input(NUMERIC, "weights") { description = "Weights array. May be null. If null, a weight of 1.0 is used" }
+        Arg(LOSS_REDUCE, "lossReduce") { description = "Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT"; defaultValue = LossReduce.MEAN_BY_NONZERO_WEIGHT_COUNT}
+        Arg(FLOATING_POINT, "delta") { description = "Loss function delta value" }
+        Output(NUMERIC, "output"){ description = "Huber loss" }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                Huber loss function, used for robust regression. It is similar both squared error loss and absolute difference loss,
+                though is less sensitive to outliers than squared error.<br>
+                Huber loss implements:
+                <pre>
+                {@code L = 0.5 * (label[i] - predictions[i])^2 if abs(label[i] - predictions[i]) < delta}
+                {@code L = delta * abs(label[i] - predictions[i]) - 0.5 * delta^2 otherwise}
+                </pre>
+            """.trimIndent()
+        }
+    }
+
+    Op("l2Loss") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.loss"
+        javaOpClass = "L2Loss"
+        Input(NUMERIC, "var") { description = "Variable to calculate L2 loss of" }
+        Output(NUMERIC, "output"){ description = "L2 loss" }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                L2 loss: 1/2 * sum(x^2)
+            """.trimIndent()
+        }
+    }
+
+    Op("logLoss") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.loss"
+        javaOpClass = "LogLoss"
+        Input(NUMERIC, "label") { description = "Label array" }
+        Input(NUMERIC, "predictions") { description = "Predictions array" }
+        Input(NUMERIC, "weights") { description = "Weights array. May be null. If null, a weight of 1.0 is used"; defaultValue = null }
+        Arg(LOSS_REDUCE, "lossReduce") { description = "Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT"; defaultValue = LossReduce.MEAN_BY_NONZERO_WEIGHT_COUNT}
+        Arg(FLOATING_POINT, "epsilon") { description = "epsilon"; defaultValue = 0.0 }
+        Output(NUMERIC, "output"){ description = "Log loss " }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                Log loss, i.e., binary cross entropy loss, usually used for binary multi-label classification. Implements:
+                {@code -1/numExamples * sum_i (labels[i] * log(predictions[i] + epsilon) + (1-labels[i]) * log(1-predictions[i] + epsilon))}
+            """.trimIndent()
+        }
+    }
+
+    Op("logPoisson") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.loss"
+        javaOpClass = "LogPoissonLoss"
+        Input(NUMERIC, "label") { description = "Label array. Each value should be 0.0 or 1.0" }
+        Input(NUMERIC, "predictions") { description = "Predictions array (has to be log(x) of actual predictions)" }
+        Input(NUMERIC, "weights") { description = "Weights array. May be null. If null, a weight of 1.0 is used" }
+        Arg(LOSS_REDUCE, "lossReduce") { description = "Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT"; defaultValue = LossReduce.MEAN_BY_NONZERO_WEIGHT_COUNT}
+        Arg(BOOL, "full") {description = "Boolean flag. true for logPoissonFull, false for logPoisson"}
+        Output(NUMERIC, "output"){ description = "Loss variable" }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                Log poisson loss: a loss function used for training classifiers.
+                Implements {@code L = exp(c) - z * c} where c is log(predictions) and z is labels.
+            """.trimIndent()
+        }
+    }
+
+    // logPoissonFull is not implemented. Simply a moniker for logPoisson with full = true
+
+    Op("meanPairwiseSquaredError") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.loss"
+        javaOpClass = "MeanPairwiseSquaredErrorLoss"
+        Input(NUMERIC, "label") { description = "Label array" }
+        Input(NUMERIC, "predictions") { description = "Predictions array" }
+        Input(NUMERIC, "weights") { description = "Weights array. May be null. If null, a weight of 1.0 is used. Must be either null, scalar, or have shape [batchSize]" }
+        Arg(LOSS_REDUCE, "lossReduce") { description = "Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT"; defaultValue = LossReduce.MEAN_BY_NONZERO_WEIGHT_COUNT}
+        Output(NUMERIC, "output"){ description = "Loss variable, scalar output" }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                Mean pairwise squared error.<br>
+                MPWSE loss calculates the difference between pairs of consecutive elements in the predictions and labels arrays.
+                For example, if predictions = [p0, p1, p2] and labels are [l0, l1, l2] then MPWSE is:
+                {@code [((p0-p1) - (l0-l1))^2 + ((p0-p2) - (l0-l2))^2 + ((p1-p2) - (l1-l2))^2] / 3}<br>
+            """.trimIndent()
+        }
+    }
+
+    Op("meanSquaredError") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.loss"
+        javaOpClass = "MeanSquaredErrorLoss"
+        Input(NUMERIC, "label") { description = "Label array" }
+        Input(NUMERIC, "predictions") { description = "Predictions array" }
+        Input(NUMERIC, "weights") { description = "Weights array. May be null. If null, a weight of 1.0 is used" }
+        Arg(LOSS_REDUCE, "lossReduce") { description = "Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT"; defaultValue = LossReduce.MEAN_BY_NONZERO_WEIGHT_COUNT}
+        Output(NUMERIC, "output"){ description = "Loss variable" }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                Mean squared error loss function. Implements {@code (label[i] - prediction[i])^2} - i.e., squared error on a per-element basis.
+                When averaged (using LossReduce#MEAN_BY_WEIGHT or LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT (the default))
+                this is the mean squared error loss function.
+            """.trimIndent()
+        }
+    }
+
+    Op("sigmoidCrossEntropy") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.loss"
+        javaOpClass = "SigmoidCrossEntropyLoss"
+        Input(NUMERIC, "label") { description = "Label array" }
+        Input(NUMERIC, "predictionLogits") { description = "Predictions array" }
+        Input(NUMERIC, "weights") { description = "Weights array. May be null. If null, a weight of 1.0 is used" }
+        Arg(LOSS_REDUCE, "lossReduce") { description = "Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT"; defaultValue = LossReduce.MEAN_BY_NONZERO_WEIGHT_COUNT}
+        Arg(FLOATING_POINT, "labelSmoothing") { description = "Label smoothing value. Default value: 0"; defaultValue = 0.0}
+        Output(NUMERIC, "output"){ description = "Loss variable" }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                Sigmoid cross entropy: applies the sigmoid activation function on the input logits (input "pre-sigmoid preductions")
+                and implements the binary cross entropy loss function. This implementation is numerically more stable than using
+                standard (but separate) sigmoid activation function and log loss (binary cross entropy) loss function.<br>
+                Implements:
+                {@code -1/numExamples * sum_i (labels[i] * log(sigmoid(logits[i])) + (1-labels[i]) * log(1-sigmoid(logits[i])))}
+                though this is done in a mathematically equivalent but more numerical stable form.<br>
+                <br>
+                When label smoothing is > 0, the following label smoothing is used:<br>
+                <pre>
+                {@code numClasses = labels.size(1);
+                label = (1.0 - labelSmoothing) * label + 0.5 * labelSmoothing}
+                </pre>
+            """.trimIndent()
+        }
+    }
+
+    Op("softmaxCrossEntropy") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.loss"
+        javaOpClass = "SoftmaxCrossEntropyLoss"
+        Input(NUMERIC, "oneHotLabels") { description = "Label array. Should be one-hot per example and same shape as predictions (for example, [mb, nOut])" }
+        Input(NUMERIC, "logitPredictions") { description = "Predictions array (pre-softmax)" }
+        Input(NUMERIC, "weights") { description = "Weights array. May be null. If null, a weight of 1.0 is used" }
+        Arg(LOSS_REDUCE, "lossReduce") { description = "Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT"; defaultValue = LossReduce.MEAN_BY_NONZERO_WEIGHT_COUNT}
+        Arg(FLOATING_POINT, "labelSmoothing") { description = "Label smoothing value. Default value: 0"; defaultValue = 0.0}
+        Output(NUMERIC, "output"){ description = "Loss variable" }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                Applies the softmax activation function to the input, then implement multi-class cross entropy:<br>
+                {@code -sum_classes label[i] * log(p[c])} where {@code p = softmax(logits)}<br>
+                If LossReduce#NONE is used, returned shape is [numExamples] out for [numExamples, numClasses] predicitons/labels;
+                otherwise, the output is a scalar.<br>
+                <p>
+                When label smoothing is > 0, the following label smoothing is used:<br>
+                <pre>
+                {@code numClasses = labels.size(1);
+                oneHotLabel = (1.0 - labelSmoothing) * oneHotLabels + labelSmoothing/numClasses}
+                </pre>
+            """.trimIndent()
+        }
+    }
+
+    Op("sparseSoftmaxCrossEntropy") {
+        javaPackage = "org.nd4j.linalg.api.ops.impl.loss"
+        javaOpClass = "SparseSoftmaxCrossEntropyLossWithLogits"
+        Input(NUMERIC, "logits") { description = "Logits array (\"pre-softmax activations\")" }
+        Input(INT, "labels") { description = "Labels array. Must be an integer type." }
+        Output(NUMERIC, "output"){ description = "Softmax cross entropy" }
+        Doc(Language.ANY, DocScope.ALL){
+            """
+                As per softmaxCrossEntropy(String, SDVariable, SDVariable, LossReduce) but the labels variable
+                is represented as an integer array instead of the equivalent one-hot array.<br>
+                i.e., if logits are rank N, then labels have rank N-1
+            """.trimIndent()
+        }
+    }
+
+}

codegen/op-codegen/src/main/resources/logback.xml

@@ -0,0 +1,51 @@
+<!--
+  ~ /* ******************************************************************************
+  ~  *
+  ~  *
+  ~  * This program and the accompanying materials are made available under the
+  ~  * terms of the Apache License, Version 2.0 which is available at
+  ~  * https://www.apache.org/licenses/LICENSE-2.0.
+  ~  *
+  ~  *  See the NOTICE file distributed with this work for additional
+  ~  *  information regarding copyright ownership.
+  ~  * Unless required by applicable law or agreed to in writing, software
+  ~  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  ~  * License for the specific language governing permissions and limitations
+  ~  * under the License.
+  ~  *
+  ~  * SPDX-License-Identifier: Apache-2.0
+  ~  ******************************************************************************/
+  -->
+
+<configuration>
+
+	
+
+	<appender name="FILE" class="ch.qos.logback.core.FileAppender">
+		<file>logs/application.log</file>
+		<encoder>
+			<pattern> %logger{15} - %message%n%xException{5}
+			</pattern>
+		</encoder>
+	</appender>
+
+	<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+		<encoder>
+			<pattern> %logger{15} - %message%n%xException{5}
+			</pattern>
+		</encoder>
+	</appender>
+
+    <logger name="org.apache.catalina.core" level="DEBUG" />
+	<logger name="org.springframework" level="WARN" />
+	<logger name="org.nd4j" level="INFO" />
+	<logger name="org.deeplearning4j" level="INFO" />
+
+
+	<root level="ERROR">
+		<appender-ref ref="STDOUT" />
+		<appender-ref ref="FILE" />
+	</root>
+
+</configuration>

codegen/op-codegen/src/main/resources/namespaces/math.json

@@ -0,0 +1,54 @@
+{ "name" : "math",
+
+  "include" : [
+
+  ],
+
+  "ops": [
+
+  {
+    "opName" : "BaseArithmeticOp",
+    "isAbstract" : true,
+    "javaPackage" : "org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic",
+    "inputs" : [ {
+      "name" : "x",
+      "description" : "First input to %OPNAME%",
+      "constraints" : ["T"]
+    }, {
+      "name" : "y",
+      "description" : "Second input to %OPNAME%",
+      "constraints" : ["T"]
+    } ],
+    "outputs" : [ {
+      "name" : "z",
+      "description" : "Output array after executing %OPNAME% on inputs"
+    } ],
+    "args" : null,
+    "constraints" : {
+      "T": {
+        "type": "allowed_dtype",
+        "values": [
+          "NUMERICAL"
+        ]
+      }
+    },
+    "doc" : [ {
+      "scope" : "ALL",
+      "language" : "ANY",
+      "text" : "%OPNAME% op doc text that will appear everywhere - classes, constructors, op creators"
+    } ]
+  },
+
+
+  {
+    "opName" : "Add",
+    "libnd4jOpName" : "add",
+    "extendsOp" : "BaseArithmeticOp"
+  },
+
+  {
+    "opName" : "Sub",
+    "libnd4jOpName" : "sub",
+    "extendsOp" : "BaseArithmeticOp"
+  }
+]}

codegen/op-codegen/src/test/java/org/nd4j/codegen/dsl/DocsGeneratorTest.java

@@ -0,0 +1,41 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+package org.nd4j.codegen.dsl;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.jupiter.api.Test;
+import org.nd4j.codegen.impl.java.DocsGenerator;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@DisplayName("Docs Generator Test")
+class DocsGeneratorTest {
+
+    @Test
+    @DisplayName("Test J Dto MD Adapter")
+    void testJDtoMDAdapter() {
+        String original = "{@code %INPUT_TYPE% eye = eye(3,2)\n" + "                eye:\n" + "                [ 1, 0]\n" + "                [ 0, 1]\n" + "                [ 0, 0]}";
+        String expected = "{ INDArray eye = eye(3,2)\n" + "                eye:\n" + "                [ 1, 0]\n" + "                [ 0, 1]\n" + "                [ 0, 0]}";
+        DocsGenerator.JavaDocToMDAdapter adapter = new DocsGenerator.JavaDocToMDAdapter(original);
+        String out = adapter.filter("@code", StringUtils.EMPTY).filter("%INPUT_TYPE%", "INDArray").toString();
+        assertEquals(out, expected);
+    }
+}

codegen/op-codegen/src/test/java/org/nd4j/codegen/dsl/TestGeneration.java

@@ -0,0 +1,67 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.dsl;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.nd4j.codegen.api.NamespaceOps;
+import org.nd4j.codegen.impl.java.Nd4jNamespaceGenerator;
+import org.nd4j.codegen.ops.RNNKt;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+class TestGeneration {
+
+    @SuppressWarnings("unused")
+    @TempDir
+    public File testDir;
+
+    @Test
+    void test() throws Exception {
+        File f = testDir;
+
+//        List<NamespaceOps> list = Arrays.asList(BitwiseKt.Bitwise(), RandomKt.Random());
+        List<NamespaceOps> list = Arrays.asList(RNNKt.SDRNN());
+
+        for(NamespaceOps ops : list) {
+            Nd4jNamespaceGenerator.generate(ops, null, f, ops.getName() + ".java", "org.nd4j.linalg.factory", StringUtils.EMPTY);
+        }
+
+        File[] files = f.listFiles();
+        Iterator<File> iter = FileUtils.iterateFiles(f, null, true);
+        if(files != null) {
+            while(iter.hasNext()){
+                File file = iter.next();
+                if(file.isDirectory())
+                    continue;
+                System.out.println(FileUtils.readFileToString(file, StandardCharsets.UTF_8));
+                System.out.println("\n\n================\n\n");
+            }
+        }
+    }
+
+}

codegen/op-codegen/src/test/kotlin/org/nd4j/codegen/dsl/ConfigTest.kt

@@ -0,0 +1,64 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.dsl
+
+import org.junit.jupiter.api.Test
+import org.nd4j.codegen.api.DataType.FLOATING_POINT
+import org.nd4j.codegen.api.Language
+import org.nd4j.codegen.api.doc.DocScope
+
+class ConfigTest {
+    @Test
+    fun allGood(){
+        Namespace("RNN"){
+            val sruWeights = Config("SRUWeights"){
+                Input(FLOATING_POINT, "weights"){ description = "Weights, with shape [inSize, 3*inSize]" }
+                Input(FLOATING_POINT, "bias"){ description = "Biases, with shape [2*inSize]" }
+            }
+
+            Op("SRU"){
+                Input(FLOATING_POINT, "x"){ description = "..." }
+                Input(FLOATING_POINT, "initialC"){ description = "..." }
+                Input(FLOATING_POINT, "mask"){ description = "..." }
+
+                useConfig(sruWeights)
+
+                Output(FLOATING_POINT, "out"){ description = "..." }
+
+                Doc(Language.ANY, DocScope.ALL){ "some doc" }
+            }
+
+            Op("SRUCell"){
+                val x = Input(FLOATING_POINT, "x"){ description = "..." }
+                val cLast = Input(FLOATING_POINT, "cLast"){ description = "..." }
+
+                val conf = useConfig(sruWeights)
+
+                Output(FLOATING_POINT, "out"){ description = "..." }
+
+                // Just for demonstration purposes
+                Signature(x, cLast, conf)
+
+                Doc(Language.ANY, DocScope.ALL){ "some doc" }
+            }
+        }
+    }
+}

codegen/op-codegen/src/test/kotlin/org/nd4j/codegen/dsl/ConstraintTest.kt

@@ -0,0 +1,99 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.dsl
+
+import org.junit.jupiter.api.Test
+import org.nd4j.codegen.api.Arg
+import org.nd4j.codegen.api.DataType
+import org.nd4j.codegen.api.Expression
+import org.nd4j.codegen.api.Input
+import org.nd4j.codegen.impl.java.JavaConstraintCodeGenerator
+import kotlin.test.assertEquals
+
+
+class ConstraintTest {
+
+
+    private fun buildConstraint(block: ConstraintBuilder.() -> Expression): Expression {
+        return ConstraintBuilder().block()
+    }
+
+    @Test
+    fun simpleConstraintTest() {
+        val expected = "x.rank() == 3"
+        val input = Input(name = "x", type = DataType.INT)
+        val constraint = buildConstraint { input.rank() eq 3 }
+        val out = JavaConstraintCodeGenerator().generateExpression(constraint)
+        assertEquals(expected, out)
+    }
+
+    @Test
+    fun simple2ConstraintTest() {
+        val expected = "(x.rank() == 3) && (x.sizeAt(2) >= 7)"
+        val input = Input(name = "x", type = DataType.INT)
+        val constraint = buildConstraint { (input.rank() eq 3) and (input.sizeAt(2) gte 7) }
+        val out = JavaConstraintCodeGenerator().generateExpression(constraint)
+        assertEquals(expected, out)
+    }
+
+    @Test
+    fun simple3ConstraintTest() {
+        val expected = "((x.rank() == 3) || (x.sizeAt(2) >= 7)) || (x.sizeAt(4) < 5)"
+        val input = Input(name = "x", type = DataType.INT)
+        val constraint = buildConstraint { some(
+                input.rank() eq 3,
+                input.sizeAt(2) gte 7,
+                input.sizeAt(4) lt 5
+        ) }
+        val out = JavaConstraintCodeGenerator().generateExpression(constraint)
+        assertEquals(expected, out)
+    }
+
+    @Test
+    fun complexConstraintTest() {
+        val expected = "(x.rank() == 3) == false"
+        val input = Input(name = "x", type = DataType.INT)
+        val constraint = buildConstraint { not(input.rank() eq 3) }
+        val out = JavaConstraintCodeGenerator().generateExpression(constraint)
+        assertEquals(expected, out)
+    }
+
+    @Test
+    fun argConstraintTest() {
+        val expected = "(x.rank() == rank) == false"
+        val arg = Arg(name = "rank", type = DataType.NUMERIC)
+        val input = Input(name = "x", type = DataType.INT)
+        val constraint = buildConstraint { not(input.rank() eq arg) }
+        val out = JavaConstraintCodeGenerator().generateExpression(constraint)
+        assertEquals(expected, out)
+    }
+
+    @Test
+    fun specificConstraintTest(){
+        val expected = "isSameType(x, y, z)"
+        val x = Input(name = "x", type = DataType.INT)
+        val y = Input(name = "y", type = DataType.INT)
+        val z = Input(name = "z", type = DataType.INT)
+        val constraint = buildConstraint { sameType(x, y, z) }
+        val out = JavaConstraintCodeGenerator().generateExpression(constraint)
+        assertEquals(expected, out)
+    }
+}

codegen/op-codegen/src/test/kotlin/org/nd4j/codegen/dsl/NamespaceInvariantTest.kt

@@ -0,0 +1,41 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.dsl
+
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertThrows
+import org.nd4j.codegen.api.DataType
+
+class NamespaceInvariantTest {
+    @Test
+    fun checkForUnusedConfigs(){
+        val thrown = assertThrows<IllegalStateException> {
+            Namespace("RNN"){
+                Config("SRUWeights"){
+                    Input(DataType.FLOATING_POINT, "weights"){ description = "Weights, with shape [inSize, 3*inSize]" }
+                    Input(DataType.FLOATING_POINT, "bias"){ description = "Biases, with shape [2*inSize]" }
+                }
+            }
+        }
+        assertEquals("Found unused configs: SRUWeights", thrown.message)
+    }
+}

codegen/op-codegen/src/test/kotlin/org/nd4j/codegen/dsl/OpBuilderTest.kt

@@ -0,0 +1,171 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.dsl
+
+import org.apache.commons.io.FileUtils
+import org.junit.jupiter.api.Test
+import org.nd4j.codegen.api.AtLeast
+import org.nd4j.codegen.api.AtMost
+import org.nd4j.codegen.api.DataType.*
+import org.nd4j.codegen.api.Exactly
+import org.nd4j.codegen.api.Language
+import org.nd4j.codegen.api.doc.DocScope
+import org.nd4j.codegen.impl.java.JavaPoetGenerator
+import java.io.File
+import java.nio.charset.StandardCharsets
+import kotlin.test.assertTrue
+
+class OpBuilderTest {
+    private var testDir = createTempDir()
+
+    @Test
+    fun opBuilderTest() {
+        val outDir = testDir
+
+        val mathNs = Namespace("math") {
+            val config = Config("bla"){
+                val a = Input(NUMERIC, "a") { description = "This is A!"}
+                Input(NUMERIC, "c") { count = AtLeast(1); description = "This is C!"}
+                Input(NUMERIC, "e") { defaultValue = a}
+                val b = Arg(NUMERIC, "b") { description = "This is B!"}
+                Arg(NUMERIC, "d") { count = AtMost(7); description = "This is D!"}
+                Arg(NUMERIC, "f") { defaultValue = 12}
+
+                Constraint("Some constraint"){
+                    a.isScalar()
+                }
+                Constraint("Some different constraint"){
+                    b eq 7
+                }
+
+                Doc(Language.JAVA, DocScope.ALL){
+                    "This is some config documentation"
+                }
+            }
+            Op("add") {
+                javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic"
+
+                val x = Input(NUMERIC, "x") { description = "First input to add" }
+                Input(NUMERIC,"y") { count = AtLeast(1); description = "Second input to add"; defaultValue = x }
+                Arg(INT,"shape") { count = AtLeast(1); description = "shape"; defaultValue = intArrayOf(1,2,3) }
+
+
+                Output(NUMERIC, "z") { description = "Output (x+y)" }
+
+
+                Doc(Language.ANY, DocScope.ALL) {
+                    """
+                    (From AddOp) Add op doc text that will appear everywhere - classes, constructors, op creators
+                    """.trimIndent()
+                }
+                Doc(Language.ANY, DocScope.CLASS_DOC_ONLY) {
+                    "Add op doc text that will appear in all class docs (javadoc etc)"
+                }
+                Doc(Language.ANY, DocScope.CONSTRUCTORS_ONLY) {
+                    "Add op doc text for constructors only"
+                }
+
+            }
+
+            val baseArithmeticOp = Mixin("BaseArithmeticOp") {
+                javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic"
+
+                Input(NUMERIC,"x") { count = Exactly(1); description = "First operand to %OPNAME%" }
+                Input(NUMERIC,"y") { count = Exactly(1); description = "Second operand" }
+
+
+                Output(NUMERIC,"z") { description = "Output" }
+
+                Doc(Language.ANY, DocScope.ALL) {
+                    "(From BaseArithmeticOp) op doc text that will appear everywhere - classes, constructors, op creators"
+                }
+                Doc(Language.ANY, DocScope.CLASS_DOC_ONLY) {
+                    "op doc text that will appear in all class docs (javadoc etc)"
+                }
+                Doc(Language.ANY, DocScope.CONSTRUCTORS_ONLY) {
+                    "op doc text for constructors only"
+                }
+
+            }
+
+            Op("sub", extends = baseArithmeticOp)
+
+            Op("mul", extends = baseArithmeticOp)
+
+            Op("rsub", extends = baseArithmeticOp) {
+                isAbstract = false
+                javaPackage = "org.nd4j.some.other.package"
+                Doc(Language.ANY, DocScope.CREATORS_ONLY) {
+                    "(From rsub) This doc section will appear only in creator method for %OPNAME%"
+                }
+            }
+
+            Op("div"){
+                javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic"
+
+                val x = Input(NUMERIC,"x") { description = "First operand to div" }
+                val y = Input(NUMERIC,"y") { description = "Second operand to div" }
+                val idx = Arg(INT, "idx") { description = "Some kind of Index" }
+                Constraint("Compatible Rank"){
+                    x.rank() eq idx
+                }
+
+                Constraint("Compatible Shapes"){
+                    sameShape(x,y)
+                }
+
+
+                Output(NUMERIC,"z") { description = "Output" }
+
+                Doc(Language.ANY, DocScope.ALL) {
+                    "op doc text that will appear everywhere - classes, constructors, op creators"
+                }
+            }
+
+            Op("zfoo"){
+                javaPackage = "bar"
+                javaOpClass = "FooBarOp"
+                val x = Input(NUMERIC,"x") { description = "First operand to %OPNAME% (%INPUT_TYPE%)" }
+                val y = Input(NUMERIC,"y") { description = "Second operand to div" }
+                val z = Arg(ENUM, "fooMode"){ description = "Something or other"; possibleValues = listOf("SOME", "value", "Spam", "eGGs")}
+                val bla = useConfig(config)
+
+                Constraint("foo bar"){
+                    x.sizeAt(7) eq 7 and y.isScalar()
+                }
+
+                Doc(Language.ANY, DocScope.ALL) {
+                    "op doc text that will appear everywhere - classes, constructors, op creators"
+                }
+
+                Signature(x, y, z, bla)
+            }
+        }
+
+        val generator = JavaPoetGenerator()
+        generator.generateNamespaceNd4j(mathNs, null, outDir, "Nd4jMath")
+        val exp = File(outDir, "org/nd4j/linalg/factory/ops/Nd4jMath.java")
+        assertTrue(exp.isFile)
+
+        println(FileUtils.readFileToString(exp, StandardCharsets.UTF_8))
+
+    }
+}

codegen/op-codegen/src/test/kotlin/org/nd4j/codegen/dsl/OpInvariantTest.kt

@@ -0,0 +1,833 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.dsl
+
+import org.junit.jupiter.api.Assertions.assertSame
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertThrows
+import org.nd4j.codegen.api.*
+import org.nd4j.codegen.api.doc.DocScope
+import kotlin.test.assertEquals
+import kotlin.test.assertNotSame
+
+
+class OpInvariantTest {
+
+    @Test
+    fun opMustBeDocumented() {
+        val thrown = assertThrows<java.lang.IllegalStateException> {
+            Namespace("math") {
+                Op("foo") {}
+            }
+        }
+        assertEquals("foo: Ops must be documented!", thrown.message)
+    }
+
+
+    @Test
+    fun opMustBeDocumentedAndNotEmpty() {
+        val thrown = assertThrows<java.lang.IllegalStateException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "" }
+                }
+            }
+        }
+        assertEquals("foo: Ops must be documented!", thrown.message)
+    }
+
+    @Test
+    fun opMustBeDocumentedWithDoc() {
+        Namespace("math") {
+            Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+            }
+        }
+    }
+
+    @Test
+    fun opSignatureMustCoverAllParameters() {
+        val thrown = assertThrows<java.lang.IllegalStateException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    val x = Input(DataType.NUMERIC, "x")
+                    val y = Input(DataType.NUMERIC, "y")
+
+                    Signature(x)
+                }
+            }
+        }
+        assertEquals("foo: Signature(x) does not cover all parameters! Missing: y", thrown.message)
+    }
+
+    @Test
+    fun opSignatureMustCoverAllParameters2() {
+        val thrown = assertThrows<java.lang.IllegalStateException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    val x = Input(DataType.NUMERIC, "x")
+                    val y = Arg(DataType.NUMERIC, "y")
+
+                    Signature(x)
+                }
+            }
+        }
+
+        assertEquals("foo: Signature(x) does not cover all parameters! Missing: y", thrown.message)
+    }
+
+    @Test
+    fun opSignatureMustCoverAllParametersWithoutDefaults() {
+        Namespace("math") {
+            Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val x = Input(DataType.NUMERIC, "x")
+                val y = Arg(DataType.NUMERIC, "y") {
+                    defaultValue = 7
+                }
+
+                Signature(x)
+            }
+        }
+    }
+
+    @Test
+    fun opSignatureMustTakeEachParameterOnlyOnce() {
+        val thrown = assertThrows<java.lang.IllegalArgumentException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    val x = Input(DataType.NUMERIC, "x")
+                    val y = Arg(DataType.NUMERIC, "y")
+
+                    Signature(x, x, x)
+                }
+            }
+        }
+
+        assertEquals("A parameter may not be used twice in a signature!", thrown.message)
+    }
+
+    @Test
+    fun opSignatureMustAllowOutputs() {
+        Namespace("math") {
+            Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val x = Input(DataType.NUMERIC, "x")
+                val y = Arg(DataType.NUMERIC, "y") {
+                    defaultValue = 7
+                }
+                val out = Output(DataType.NUMERIC, "out")
+
+                Signature(out, x)
+            }
+        }
+    }
+
+    @Test
+    fun opSignatureMustAllowOutputsOnlyOnce() {
+        val thrown = assertThrows<java.lang.IllegalArgumentException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    val x = Input(DataType.NUMERIC, "x")
+                    val y = Arg(DataType.NUMERIC, "y") {
+                        defaultValue = 7
+                    }
+                    val out = Output(DataType.NUMERIC, "out")
+
+                    Signature(out, x, out)
+                }
+            }
+        }
+
+        assertEquals("A parameter may not be used twice in a signature!", thrown.message)
+    }
+
+    @Test
+    fun opSignatureDefaultValueMustHaveCorrectShape() {
+        val thrown = assertThrows<java.lang.IllegalArgumentException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    val out = Output(DataType.NUMERIC, "out")
+                    val x = Input(DataType.NUMERIC, "x")
+                    val y = Arg(DataType.INT, "y") {
+                        defaultValue = x.shape()
+                    }
+
+                    Signature(x)
+                }
+            }
+        }
+
+        assertEquals("Illegal default value for Arg(INT, y). Got x.shape() (org.nd4j.codegen.api.TensorShapeValue)", thrown.message)
+    }
+
+    @Test
+    fun opSignatureDefaultValue() {
+        Namespace("math") {
+            Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x")
+                val y = Arg(DataType.INT, "y") {
+                    defaultValue = 2
+                }
+
+                Signature(x)
+            }
+        }
+    }
+
+    @Test
+    fun opSignatureDefaultValueMustHaveCorrectDataType() {
+        val thrown = assertThrows<java.lang.IllegalArgumentException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    val out = Output(DataType.NUMERIC, "out")
+                    val x = Input(DataType.NUMERIC, "x")
+                    val y = Arg(DataType.INT, "y") {
+                        defaultValue = 1.7
+                    }
+
+                    Signature(x)
+                }
+            }
+        }
+
+        assertEquals("Illegal default value for Arg(INT, y). Got 1.7 (java.lang.Double)", thrown.message)
+    }
+
+
+    @Test
+    fun opSignatureDefaultInputReference() {
+        val thrown = assertThrows<java.lang.IllegalStateException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    val out = Output(DataType.NUMERIC, "out")
+                    val x = Input(DataType.NUMERIC, "x")
+                    val z = Input(DataType.NUMERIC, "z")
+                    val y = Arg(DataType.INT, "y") {
+                        count = AtLeast(1)
+                        defaultValue = z.shape()
+                    }
+
+                    Signature(x)
+                }
+            }
+        }
+
+        assertEquals("foo: Signature(x) does not cover all parameters! Missing: z, y", thrown.message)
+    }
+
+    @Test
+    fun opSignatureDefaultOutputReference() {
+        val thrown = assertThrows<java.lang.IllegalStateException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    val out = Output(DataType.NUMERIC, "out")
+                    val x = Input(DataType.NUMERIC, "x")
+                    val y = Arg(DataType.INT, "y") {
+                        count = AtLeast(1)
+                        defaultValue = out.shape()
+                    }
+
+                    Signature(x)
+                }
+            }
+        }
+
+        assertEquals("foo: Signature(x) does not cover all parameters! Missing: y", thrown.message)
+    }
+
+    @Test
+    fun opSignatureDefaultWithOutputReference() {
+        Namespace("math") {
+            Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x")
+                val y = Arg(DataType.INT, "y") {
+                    count = AtLeast(1)
+                    defaultValue = out.shape()
+                }
+
+                Signature(out, x)
+            }
+        }
+    }
+
+    @Test
+    fun opSignatureDefaultReferenceChain() {
+        val thrown = assertThrows<java.lang.IllegalStateException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    val out = Output(DataType.NUMERIC, "out")
+                    val x = Input(DataType.NUMERIC, "x")
+                    val z = Input(DataType.NUMERIC, "z")
+                    val u = Input(DataType.NUMERIC, "u") { defaultValue = z }
+                    val v = Input(DataType.NUMERIC, "v") { defaultValue = u }
+                    val y = Arg(DataType.INT, "y") {
+                        count = AtLeast(1)
+                        defaultValue = v.shape()
+                    }
+
+                    Signature(x)
+                }
+            }
+        }
+
+        assertEquals("foo: Signature(x) does not cover all parameters! Missing: z, u, v, y", thrown.message)
+    }
+
+    @Test
+    fun opSignatureDefaultReferenceChainWorking() {
+        Namespace("math") {
+            Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x")
+                val z = Input(DataType.NUMERIC, "z") { defaultValue = x }
+                val u = Input(DataType.NUMERIC, "u") { defaultValue = z }
+                val v = Input(DataType.NUMERIC, "v") { defaultValue = u }
+                val y = Arg(DataType.INT, "y") {
+                    count = AtLeast(1)
+                    defaultValue = v.shape()
+                }
+
+                Signature(x)
+            }
+        }
+    }
+
+    @Test
+    fun opSignatureShorthandAllParams() {
+        Namespace("math") {
+            Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x")
+                val z = Input(DataType.NUMERIC, "z") { defaultValue = x }
+                val u = Input(DataType.NUMERIC, "u") { defaultValue = z }
+                val v = Input(DataType.NUMERIC, "v") { defaultValue = u }
+                val y = Arg(DataType.INT, "y") {
+                    count = AtLeast(1)
+                    defaultValue = v.shape()
+                }
+
+                AllParamSignature()
+            }
+        }
+
+    }
+
+    @Test
+    fun opSignatureNullDefaults() {
+        Namespace("math") {
+            Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x")
+                val y = Arg(DataType.INT, "y") {
+                    count = AtLeast(1)
+                    defaultValue = null
+                }
+
+                AllDefaultsSignature()
+            }
+        }
+    }
+
+    @Test
+    fun opSignatureNullDefaultsForInputs() {
+        Namespace("math") {
+            Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x") { defaultValue = null }
+
+                AllDefaultsSignature()
+            }
+        }
+    }
+
+    @Test
+    fun opSignatureShorthandDefaultParams() {
+        Namespace("math") {
+            Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x")
+                val z = Input(DataType.NUMERIC, "z") { defaultValue = x }
+                val u = Input(DataType.NUMERIC, "u") { defaultValue = z }
+                val v = Input(DataType.NUMERIC, "v") { defaultValue = u }
+                val y = Arg(DataType.INT, "y") {
+                    count = AtLeast(1)
+                    defaultValue = v.shape()
+                }
+
+                AllDefaultsSignature()
+            }
+        }
+    }
+
+    @Test
+    fun opSignatureSupportsArrayDefaults() {
+        Namespace("math") {
+            Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x")
+                val y = Arg(DataType.INT, "y") { count = AtLeast(0); defaultValue = intArrayOf() }
+                val z = Arg(DataType.FLOATING_POINT, "z") { count = Range(2, 5); defaultValue = doubleArrayOf(1.0, 2.0, 3.0) }
+                val a = Arg(DataType.BOOL, "a") { count = AtLeast(1); defaultValue = booleanArrayOf(true) }
+
+                AllDefaultsSignature()
+            }
+        }
+    }
+
+    @Test
+    fun opSignatureSupportsArrayDefaultsAtLeast() {
+        val thrown = assertThrows<java.lang.IllegalArgumentException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    Output(DataType.NUMERIC, "out")
+                    Input(DataType.NUMERIC, "x")
+                    Arg(DataType.INT, "y") { count = AtLeast(1); defaultValue = intArrayOf() }
+                }
+            }
+        }
+
+        assertEquals("Illegal default value for Arg(INT, y){ count = AtLeast(min=1) }. Got [] ([I)", thrown.message)
+
+    }
+
+    @Test
+    fun opSignatureSupportsArrayDefaultsAtMost() {
+        val thrown = assertThrows<java.lang.IllegalArgumentException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    Output(DataType.NUMERIC, "out")
+                    Input(DataType.NUMERIC, "x")
+                    Arg(DataType.INT, "y") { count = AtMost(1); defaultValue = intArrayOf(1, 2) }
+                }
+            }
+        }
+
+        assertEquals("Illegal default value for Arg(INT, y){ count = AtMost(max=1) }. Got [1, 2] ([I)", thrown.message)
+
+    }
+
+    @Test
+    fun opSignatureSupportsArrayDefaultsRange() {
+        val thrown = assertThrows<java.lang.IllegalArgumentException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    Output(DataType.NUMERIC, "out")
+                    Input(DataType.NUMERIC, "x")
+                    Arg(DataType.INT, "y") { count = Range(3, 7); defaultValue = intArrayOf() }
+                }
+            }
+        }
+
+        assertEquals("Illegal default value for Arg(INT, y){ count = Range(from=3, to=7) }. Got [] ([I)", thrown.message)
+    }
+
+    @Test
+    fun opSignatureSupportsArrayDefaultsExactly() {
+        val thrown = assertThrows<java.lang.IllegalArgumentException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    Output(DataType.NUMERIC, "out")
+                    Input(DataType.NUMERIC, "x")
+                    Arg(DataType.INT, "y") { count = Exactly(7); defaultValue = intArrayOf() }
+                }
+            }
+        }
+
+        assertEquals("Illegal default value for Arg(INT, y){ count = Exactly(count=7) }. Got [] ([I)", thrown.message)
+
+    }
+
+    @Test
+    fun opSignatureHasExpectedNumberOfSignatures() {
+        Namespace("math") {
+            val op = Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x")
+                val y = Arg(DataType.INT, "y") { count = AtLeast(0); defaultValue = intArrayOf() }
+
+                AllParamSignature()
+                AllDefaultsSignature()
+            }
+
+            assertEquals(2, op.signatures.size)
+        }
+    }
+
+    @Test
+    fun opSignatureHasExpectedNumberOfSignaturesWithOutput() {
+        Namespace("math") {
+            val op = Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x")
+                val y = Arg(DataType.INT, "y") { count = AtLeast(0); defaultValue = intArrayOf() }
+
+                AllParamSignature(true)
+                AllDefaultsSignature(true)
+            }
+
+            assertEquals(4, op.signatures.size)
+        }
+    }
+
+    @Test
+    fun opSignatureHasExpectedNumberOfSignaturesNoDefaults() {
+        Namespace("math") {
+            val op = Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x")
+                val y = Arg(DataType.INT, "y") { count = AtLeast(0); }
+
+                AllParamSignature()
+                AllDefaultsSignature()
+            }
+
+            assertEquals(1, op.signatures.size)
+        }
+    }
+
+    @Test
+    fun opSignatureHasExpectedNumberOfSignaturesWithOutputNoDefaults() {
+        Namespace("math") {
+            val op = Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x")
+                val y = Arg(DataType.INT, "y") { count = AtLeast(0); }
+
+                AllParamSignature(true)
+                AllDefaultsSignature(true)
+            }
+
+            assertEquals(2, op.signatures.size)
+        }
+    }
+
+    @Test
+    fun argEnum() {
+        Namespace("math") {
+            Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x")
+                val y = Arg(DataType.ENUM, "y") { possibleValues = listOf("FOO", "BAR", "BAZ"); description = "Enums require some docs" }
+
+            }
+        }
+    }
+
+    @Test
+    fun argEnumDefaultValue() {
+        Namespace("math") {
+            Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x")
+                val y = Arg(DataType.ENUM, "y") {
+                    possibleValues = listOf("FOO", "BAR", "BAZ")
+                    defaultValue = "BAZ"
+                    description = "Enums require some docs"
+                }
+
+                AllDefaultsSignature()
+            }
+        }
+    }
+
+    @Test
+    fun argEnumBadDefaultValue() {
+        val thrown = assertThrows<java.lang.IllegalArgumentException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    val out = Output(DataType.NUMERIC, "out")
+                    val x = Input(DataType.NUMERIC, "x")
+                    val y = Arg(DataType.ENUM, "y") {
+                        possibleValues = listOf("FOO", "BAR", "BAZ")
+                        defaultValue = "SPAM"
+                    }
+
+                    AllDefaultsSignature()
+                }
+            }
+        }
+
+        assertEquals("Illegal default value for Arg(ENUM(FOO, BAR, BAZ), y). Got SPAM (java.lang.String)", thrown.message)
+    }
+
+    @Test
+    fun argEnumEmptyPossibleValues() {
+        val thrown = assertThrows<java.lang.IllegalArgumentException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    val out = Output(DataType.NUMERIC, "out")
+                    val x = Input(DataType.NUMERIC, "x")
+                    val y = Arg(DataType.ENUM, "y") {
+                        possibleValues = listOf()
+                    }
+
+                }
+            }
+        }
+
+        assertEquals("Arg(ENUM(null), y): Can not set empty possibleValues.", thrown.message)
+    }
+
+    @Test
+    fun argEnumBadType() {
+        val thrown = assertThrows<java.lang.IllegalArgumentException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    val out = Output(DataType.NUMERIC, "out")
+                    val x = Input(DataType.NUMERIC, "x")
+                    val y = Arg(DataType.NUMERIC, "y") {
+                        possibleValues = listOf("FOO", "BAR", "BAZ")
+                        defaultValue = "SPAM"
+                    }
+
+                    AllDefaultsSignature()
+                }
+            }
+        }
+
+        assertEquals("Arg(NUMERIC, y): Can not set possibleValues on non ENUM typed Arg.", thrown.message)
+    }
+
+    @Test
+    fun argEnumBadCount() {
+        val thrown = assertThrows<java.lang.IllegalArgumentException> {
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    val out = Output(DataType.NUMERIC, "out")
+                    val x = Input(DataType.NUMERIC, "x")
+                    val y = Arg(DataType.ENUM, "y") {
+                        count = AtLeast(1)
+                        possibleValues = listOf("FOO", "BAR", "BAZ")
+                    }
+
+                    AllDefaultsSignature()
+                }
+            }
+        }
+
+        assertEquals("Arg(ENUM(null), y): ENUM typed Arg can not be array", thrown.message)
+    }
+
+    @Test
+    fun argEnumGoodCount() {
+        Namespace("math") {
+            Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x")
+                val y = Arg(DataType.ENUM, "y") {
+                    count = Exactly(1)
+                    possibleValues = listOf("FOO", "BAR", "BAZ")
+                    description = "Enums require some docs"
+                }
+
+                AllDefaultsSignature()
+            }
+        }
+    }
+
+    @Test
+    fun onlyValidParametersAreUsedInSignaturesBadCase() {
+        val thrown = assertThrows<IllegalArgumentException> {
+            val mixin = Mixin("Bar") {
+                Input(DataType.NUMERIC, "a")
+                Arg(DataType.BOOL, "b")
+            }
+
+            Namespace("math") {
+                Op("foo") {
+                    Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                    val out = Output(DataType.NUMERIC, "out")
+                    val x = Input(DataType.NUMERIC, "x")
+
+                    Signature(out, x, mixin.input("a"), mixin.arg("b"))
+                }
+            }
+        }
+        assertEquals("You can only use parameters in a signature that are actually defined in Op(opName=foo, libnd4jOpName=foo, isAbstract=false)! Did you forget to useMixin(...) a mixin?", thrown.message)
+    }
+
+    @Test
+    fun onlyValidParametersAreUsedInSignaturesGoodCase() {
+        val mixin = Mixin("Bar") {
+            Input(DataType.NUMERIC, "a")
+            Arg(DataType.BOOL, "b")
+        }
+
+        Namespace("math") {
+            Op("foo", mixin) {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                val out = Output(DataType.NUMERIC, "out")
+                val x = Input(DataType.NUMERIC, "x")
+
+                Signature(out, x, mixin.input("a"), mixin.arg("b"))
+            }
+        }
+    }
+
+    @Test
+    fun lastMixinDefinitionWins(){
+        val mixin = Mixin("Bar") {
+            Input(DataType.NUMERIC, "a")
+            Arg(DataType.BOOL, "b")
+        }
+
+        Namespace("math") {
+            val op = Op("foo", mixin) {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                Output(DataType.NUMERIC, "out")
+                Input(DataType.NUMERIC, "a") { count=Exactly(1)}
+            }
+
+            assertNotSame(mixin.inputs.find { it.name == "a"}, op.inputs.find { it.name == "a"})
+        }
+
+    }
+
+    @Test
+    fun lastMixinDefinitionWins2(){
+        val mixin = Mixin("Bar") {
+            Input(DataType.NUMERIC, "a")
+            Arg(DataType.BOOL, "b")
+        }
+
+        Namespace("math") {
+            val op = Op("foo") {
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                Output(DataType.NUMERIC, "out")
+                Input(DataType.NUMERIC, "a")
+                useMixin(mixin)
+            }
+
+            assertSame(mixin.inputs.find { it.name == "a"}, op.inputs.find { it.name == "a"})
+        }
+
+    }
+
+    @Test
+    fun mixinDoesOnlyOverwritePropertiesIfSetNoneSetCase(){
+        val mixin = Mixin("Bar") {
+            Input(DataType.NUMERIC, "a")
+            Arg(DataType.BOOL, "b")
+        }
+
+        Namespace("math") {
+            val op = Op("foo") {
+                javaPackage = "fooPackage"
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                Output(DataType.NUMERIC, "out")
+                Input(DataType.NUMERIC, "a")
+                useMixin(mixin)
+            }
+
+            assertEquals("fooPackage", op.javaPackage)
+        }
+
+    }
+
+    @Test
+    fun mixinDoesOnlyOverwritePropertiesIfSetSetCase(){
+        val mixin = Mixin("Bar") {
+            javaPackage = "MixinPackage"
+            Input(DataType.NUMERIC, "a")
+            Arg(DataType.BOOL, "b")
+        }
+
+        Namespace("math") {
+            val op = Op("foo") {
+                javaPackage = "fooPackage"
+                Doc(Language.ANY, DocScope.ALL) { "Some Documentation" }
+                Output(DataType.NUMERIC, "out")
+                Input(DataType.NUMERIC, "a")
+                useMixin(mixin)
+            }
+
+            assertEquals("MixinPackage", op.javaPackage)
+        }
+
+    }
+
+    @Test
+    fun mixinDoesOnlyOverwritePropertiesIfSetNoneSetCaseOnMixins(){
+        val mixin = Mixin("Bar") {
+            Input(DataType.NUMERIC, "a")
+            Arg(DataType.BOOL, "b")
+        }
+
+        val op = Mixin("foo") {
+            javaPackage = "fooPackage"
+            useMixin(mixin)
+        }
+
+        assertEquals("fooPackage", op.javaPackage)
+
+    }
+
+    @Test
+    fun mixinDoesOnlyOverwritePropertiesIfSetSetCaseOnMixins(){
+        val mixin = Mixin("Bar") {
+            javaPackage = "MixinPackage"
+            Input(DataType.NUMERIC, "a")
+            Arg(DataType.BOOL, "b")
+        }
+
+        val op = Mixin("foo") {
+            javaPackage = "fooPackage"
+            useMixin(mixin)
+        }
+
+        assertEquals("MixinPackage", op.javaPackage)
+    }
+}

codegen/op-codegen/src/test/kotlin/org/nd4j/codegen/ops/ConstructionTest.kt

@@ -0,0 +1,58 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.nd4j.codegen.ops
+
+import org.junit.jupiter.api.Test
+
+/**
+ * Test that each Namespace actually constructs properly.
+ *
+ * This is allows us to utilize run-time consistency checks during the build process - if tests are enabled.
+ */
+class ConstructionTest {
+
+    @Test
+    fun bitwise() { Bitwise() }
+
+    @Test
+    fun random() { Random() }
+
+    @Test
+    fun math() { Math() }
+
+    @Test
+    fun base() { SDBaseOps() }
+
+    @Test
+    fun loss() { SDLoss() }
+
+    @Test
+    fun cnn() { SDCNN() }
+
+    @Test
+    fun rnn() { SDRNN() }
+
+    @Test
+    fun image() { SDImage() }
+
+    @Test
+    fun nn() { NN() }
+}