chore: import upstream snapshot with attribution

libnd4j/include/graph/scheme/array.fbs

@@ -0,0 +1,84 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+namespace graph;
+
+// byte order for arrays/buffers
+enum ByteOrder:byte {
+    LE,
+    BE,
+}
+
+// DataType for arrays/buffers
+enum DType:byte {
+    INHERIT,
+    BOOL,
+    FLOAT8,
+    HALF,
+    HALF2,
+    FLOAT,
+    DOUBLE,
+    INT8,
+    INT16,
+    INT32,
+    INT64,
+    UINT8,
+    UINT16,
+    UINT32,
+    UINT64,
+    QINT8,
+    QINT16,
+    BFLOAT16 = 17,
+    UTF8 = 50,
+    UTF16 = 51,
+    UTF32 = 52,
+}
+
+enum LossReduce:byte {
+  NONE,
+  SUM,
+  MEAN_BY_WEIGHT,
+  MEAN_BY_NONZERO_WEIGHT_COUNT
+}
+
+
+// this structure describe NDArray
+// Buffer chunk for large data
+table BufferChunk {
+    index:long;      // Position in the logical buffer
+    data:[byte];     // Chunk data (limited to <2GB per chunk)
+}
+
+// Extended FlatArray with support for both small and large buffers
+// as well as external storage for arrays larger than 2GB
+table FlatArray {
+    shape:[long];
+    buffer:[byte];                     // Used for small buffers (<2GB)
+    dtype:DType;
+    byteOrder:ByteOrder;
+
+    // Fields for large buffer support via chunking
+    bufferChunks:[BufferChunk];        // Used for large buffers
+    totalBufferSize:long;              // Total size when using chunks
+    
+    // Fields for external file storage
+    externalDataFilename:[string];     // Filename for externally stored data
+    isExternal:bool = false;           // Flag indicating if data is stored externally
+}
+
+root_type FlatArray;

libnd4j/include/graph/scheme/config.fbs

@@ -0,0 +1,59 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+namespace graph;
+
+enum ProfilingMode:byte {
+    NONE,   // no checks for Z values
+    NAN_PANIC, // reporting nodes with NaN values in Z
+    INF_PANIC, // reporting nodes with Inf values in Z
+    ANY_PANIC, // reporting nodes with NaN/Inf values in Z
+}
+
+enum ExecutionMode:byte {
+    SEQUENTIAL, // all operations will be executed sequentially
+    STRICT, // all operations will be following device id for execution mode selection
+    AUTO, // all operations that can be executed in parallel - will be executed in parallel
+}
+
+enum OutputMode:byte {
+   IMPLICIT, // only final nodes of graph will be returned
+   EXPLICIT, // only declared output fields
+   EXPLICIT_AND_IMPLICIT, // both options enabled
+   VARIABLE_SPACE, // whole content of VariableSpace will be returned back
+   OPTIMIZED, // same as IMPLICIT + using in-place ops execution where possible
+}
+
+enum Direction:byte {
+    FORWARD_ONLY, // inference pass
+    FORWARD_AND_BACKWARD, // not implemented yet
+    BACKWARD_ONLY, // not implemented yet
+}
+
+table FlatConfiguration {
+    id:long; // id of the graph this configuration applies to
+    executionMode:ExecutionMode; // current execution dmode
+    profilingMode:ProfilingMode; // current profiling mode
+    outputMode:OutputMode; // current output mode
+    timestats:bool; // are we gathering time info. false by default
+    footprintForward:long; // optional: amount of memory needed for FF pass
+    footprintBackward:long; // optional: amount of memory needed for BP pass
+    direction:Direction; // execution direction
+}
+
+root_type FlatConfiguration;

libnd4j/include/graph/scheme/container.fbs

@@ -0,0 +1,96 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+include "graph.fbs";
+include "array.fbs";
+
+namespace graph;
+
+// Compression type used for sections in the container
+enum CompressionType:byte {
+    NONE,       // No compression 
+    DEFLATE,    // Deflate compression
+    ZSTD        // Zstandard compression (future)
+}
+
+// Section types in the container
+enum SectionType:byte {
+    HEADER,         // Container header information
+    METADATA,       // Metadata key-value pairs
+    GRAPH,          // Graph structure (FlatGraph)
+    ARRAYS,         // Array data
+    SHARD_INFO      // Information about sharding
+}
+
+// Container file header
+table ContainerHeader {
+    format_version:int;        // Container format version
+    section_count:int;         // Number of sections in the container
+    checksum:long;             // Optional checksum for integrity verification
+}
+
+// Metadata key-value pair
+table MetadataEntry {
+    key:string;                // Metadata key
+    value:string;              // Metadata value
+}
+
+// Metadata section
+table MetadataSection {
+    entries:[MetadataEntry];   // List of metadata entries
+}
+
+// Sharding information
+table ShardInfo {
+    shard_index:int;           // Current shard index
+    shard_count:int;           // Total number of shards
+    shard_type:string;         // Type of shard (e.g., "graph", "weights")
+    base_filename:string;      // Base filename for related shards
+}
+
+// Section header
+table SectionHeader {
+    type:SectionType;          // Section type
+    size:long;                 // Size of section data in bytes
+    compression:CompressionType; // Compression type used
+    offset:long;               // Offset in the container file
+    checksum:long;             // Optional checksum for integrity verification
+}
+
+// Array entry in the arrays section
+table ArrayEntry {
+    variable_name:string;      // Name of the variable
+    array:FlatArray;           // The array data
+}
+
+// Arrays section
+table ArraysSection {
+    arrays:[ArrayEntry];       // List of arrays
+}
+
+// Container structure
+table ModelContainer {
+    header:ContainerHeader;    // Container header
+    sections:[SectionHeader];  // Section headers
+    metadata:MetadataSection;  // Metadata section
+    graph:FlatGraph;           // Graph section
+    arrays:ArraysSection;      // Arrays section
+    shard_info:ShardInfo;      // Shard information (optional)
+}
+
+root_type ModelContainer;

libnd4j/include/graph/scheme/graph.fbs

@@ -0,0 +1,75 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+include "node.fbs";
+include "config.fbs";
+include "variable.fbs";
+include "utils.fbs";
+include "result.fbs";
+include "request.fbs";
+include "array.fbs";
+
+namespace graph;
+
+table UpdaterState {
+    paramName:string;              //Name of the parameter the updater state is for
+    updaterStateKeys:[string];       //Name of the updater state key (for example: "M" or "V" for Adam updater)
+    updaterStateValues:[FlatArray];       //Value of corresponding updater state key (for example: array for "M" updater state in Adam updater)
+}
+
+table SameDiffSubInstance {
+    name:string;                    // Name/identifier of the sub-instance
+    serializedData:[ubyte];         // Serialized FlatGraph data for the sub-instance
+}
+
+table FlatGraph {
+    id:long;                      // id of the graph
+    variables:[FlatVariable];         // list of variables
+    nodes:[FlatNode];               // list of nodes
+    outputs:[IntPair];              // list of output variables or nodes, in format of IntPair.first is node Id, IntPair.second is output index of the node
+    configuration:FlatConfiguration;    // optional execution configuration
+    placeholders:[string];          // Placeholders. Used for SameDiff serialization/deserialization
+    lossVariables:[string];             // Variables that are marked as losses. Used for training.
+    trainingConfig:string;          // JSON representation of training configuration. JSON used for custom functionality
+    updaterState:[UpdaterState];      // Updater state
+
+    // New fields for metadata
+    metadataKeys:[string];              // Keys for metadata
+    metadataValues:[string];            // Values for metadata (indexed same as keys)
+
+    // New field for sub-instances
+    subInstances:[SameDiffSubInstance]; // Sub-instances of SameDiff (nested graphs)
+}
+
+
+table FlatDropRequest {
+    id:long; // id of the graph to be droppoed
+}
+
+table FlatResponse {
+    status:int; // status of operation
+}
+
+rpc_service GraphInferenceServer {
+    RegisterGraph(FlatGraph):FlatResponse;
+    ForgetGraph(FlatDropRequest):FlatResponse;
+    ReplaceGraph(FlatGraph):FlatResponse;
+    InferenceRequest(FlatInferenceRequest):FlatResult;
+}
+
+root_type FlatGraph;

libnd4j/include/graph/scheme/node.fbs

@@ -0,0 +1,68 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+include "array.fbs";
+include "utils.fbs";
+include "properties.fbs";
+
+namespace graph;
+
+
+// this structure describes single operation within graph
+table FlatNode {
+    id:int; // unique id for this node
+    name:string; // literal id of this node (optional)
+    opType:OpType; // type of this op
+    opNum:long; // for custom ops that's field for hash, for all other (legacy XYZ ops) that's actually opNum (as defined in legacy_ops.h)
+    properties:[FlatProperties]; // optional properties
+    input:[int]; // ID's of input nodes for this node !!! only used if inputPaired in unset !!!
+    inputPaired:[IntPair]; //list of input variables or nodes, in format of IntPair.first is node Id, IntPair.second is output index of the node
+    output:[int]; // ID's of connected nodes for this node
+
+    extraParams:[double]; // extra params for this op (if any)
+    extraInteger:[long]; // optional integer extra params
+    extraBools:[bool]; // optional  boolean args
+    dimensions:[int]; // optional dimension for this operation
+
+    device:int; // default is -1, which means _auto_
+
+    // fields related to Scopes
+    scope_id:int; // unique scope id, where this op belongs to
+    scope_name:string; // literal scope id, where this op belongs to
+
+    // Additional metadata for helping SameDiff deserialization
+    outputNames:[string];       //Name of the SameDiff variable names corresponding to the outputs of this op
+    opName:string;              //Used to help resolving the class. In a few cases, multiple classes/opNames are mapped to same hash, and might have different config/properties/differentiability
+
+    // output data types (optional)
+    outputTypes:[DType];
+	
+	//Scalar value - used for scalar ops. Should be single value only.
+	scalar:FlatArray;
+	
+	//Control dependencies
+	controlDeps:[string];
+	varControlDeps:[string];
+	controlDepFor:[string];
+
+	// DArgs
+	extraTypes:[DType];
+	extraStrings:[string];
+}
+
+root_type FlatNode;

libnd4j/include/graph/scheme/properties.fbs

@@ -0,0 +1,35 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+include "array.fbs";
+
+namespace graph;
+
+
+table FlatProperties {
+    name:string;    // name of this property
+    i:[int];        // integer properties
+    l:[long];       // long properties
+    d:[double];     // double properties
+    a:[FlatArray];  // array properties
+    b:[bool];       // boolean properties
+    s:[string];     // string properties
+    shape:[int];    // array shape (for i/l/d/a/b/s). Used when encoding arrays as a 1d. Null when ildabs fields are used as length 1 for primitives/non-arrays
+}
+
+root_type FlatProperties;

libnd4j/include/graph/scheme/request.fbs

@@ -0,0 +1,30 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+include "variable.fbs";
+include "config.fbs";
+
+namespace graph;
+
+table FlatInferenceRequest {
+    id:long; // id of the graph to be executed
+    variables:[FlatVariable]; // input variables to be set as input
+    configuration:FlatConfiguration; // optional configuration for this inference run
+}
+
+root_type FlatInferenceRequest;

libnd4j/include/graph/scheme/result.fbs

@@ -0,0 +1,39 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+include "node.fbs";
+include "utils.fbs";
+include "variable.fbs";
+
+namespace graph;
+
+table FlatTiming {
+    id:int; // ID of the node for this time report
+    name:string; // name of the node for this time report (if was specified)
+    timing:LongPair; // outer time, inner time
+}
+
+table FlatResult {
+    id:long; // ID of the graph generated this result
+    variables:[FlatVariable]; // variables with results
+    timing:[FlatTiming]; // timing results
+    footprintForward:long; // amount of memory used for FF pass
+    footprintBackward:long; // amount of memory used for BP pass
+}
+
+root_type FlatResult;

libnd4j/include/graph/scheme/sequence.fbs

@@ -0,0 +1,32 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+include "array.fbs";
+
+namespace graph;
+
+table SequenceItem {
+  name:string;
+  associated_variable:[FlatArray];
+}
+
+table SequenceItemRoot {
+  sequence_items:[SequenceItem];
+}
+
+root_type SequenceItemRoot;

libnd4j/include/graph/scheme/uigraphevents.fbs

@@ -0,0 +1,139 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+include "array.fbs";	//For FlatArray
+
+namespace graph;
+
+/*
+An "Event" is any value that may occur multiple times (score vs. iteration, or accuracy for example)
+All events have timestamps and iteration/epoch.
+
+Design: Given given FlatBuffers doesn't support polymorphism, each "frame" in our log file will comprise a sequence of (Event,X) pairs
+"Event" is common information/header for the frame (i.e., determines type that follows and allows decoding), and can also be used for filtering - i.e., can skip the next entry
+
+Alternatives:
+1. Have a large Event class, with all but 1 of the 'type specific' fields being null (inefficient, especially for things like scalars)
+2. Use pair of (ubyte,X) and have duplicate fields in every event subtype
+
+Types of entries that can follow an Event:
+array:FlatArray;				//Use standard/existing graph FlatArray class. Also used for scalars! (For types and also strings etc)
+arrayList:[FlatArray];			//For TensorArray and the like
+histogram:Histogram;			//Histogram class
+image:Image;					//Could just use array/FlatArray, but idea is to store more efficiently here in compressed format
+summaryStat:SummaryStatistics;	//One class for holding stuff like min/mean/max/stdev etc - more efficiently than a whole lot of separate scalar entries...
+opTiming:FlatTiming;			//Timing/profiling information about a single op execution. Use existing FlatTiming, but maybe extend if required
+hardwareState:HardwareState;	//Information about hardware at a specific point in time: CPU/GPU utilization, etc
+*/
+
+
+enum UIEventType:byte {
+	ADD_NAME,					//Used to register a name (essentialy, Map<Integer,String>.put(i,name)), so it can be referred to by index later. Saves us encoding really long names in every single frame...
+	SCALAR,
+	ARRAY,
+	ARRAY_LIST,
+	HISTOGRAM,
+	IMAGE,						//To be added later
+	SUMMARY_STATISTICS,
+	OP_TIMING,
+	HARDWARE_STATE
+}
+
+/*
+UIEventSubtype relates to what the metric is about. This determines where the value should be presented in the UI
+For example, we can have scalars for: evaluation, tuning (mean magnitudes), performance (op runtime) etc.
+But these should be presented in different sections of the UI.
+It can also be thought of as the "semantic type of event" whereas UIEventType is the "data type of event"
+*/
+enum UIEventSubtype:byte {
+	NONE,						//Not applicable (for example, ADD_NAME event type)
+	EVALUATION,					//Train/test accuracy, etc
+	LOSS,						//Value of the loss function (or any sub-component there-of, such as L2)
+	LEARNING_RATE,				//Learning rate
+	TUNING_METRIC,				//Metrics like: parameter:update ratio, or parameter and gradient histograms, etc
+	PERFORMANCE,				//Global performance metrics - batches/sec, epoch time, etc
+	PROFILING,					//Op profiling/performance metrics - how long to run each op, etc
+	FEATURE_LABEL,				//Feature/input - for visualization, debugging, etc
+	PREDICTION,					//Network prediction/output
+	USER_CUSTOM					//Custom, user-defined metric or value
+}
+
+table UIEvent {
+    eventType:UIEventType;		//Type of the event that follows
+	eventSubType:UIEventSubtype;//Subtype of event that follows
+	nameIdx:int;				//Integer representing the previously registered name of the event - for example, 0=="accuracy", 1=="score", 2=="weights" etc as previously registered
+	timestamp:long;
+	iteration:int;
+	epoch:int;
+	variableId:int16;			//Number of the variable. Optional (-1 == not applicable)
+	frameIter:FrameIteration;	//Optional - some events have a corresponding frame/iteration
+	plugin:uint16;				//An ID number - what UI page/plugin should be used to render this information? (Allows for extensibility, separation of data for different UI components)
+}
+
+//Optional, often null. Used for events that have an associated frame/iteration (like array values in a loop)
+table FrameIteration {
+	frame:string;
+	iteration:uint16;
+}
+
+//Used to register a name (essentialy, Map<Integer,String>.put(i,name)), so it can be referred to by index later. Saves us encoding really long names in every single frame...
+table UIAddName {
+	nameIdx:int;
+	name:string;
+}
+
+//A simple list of arrays
+table FlatArrayList {
+	list:[FlatArray];
+}
+
+enum UIHistogramType:byte {
+	DISCRETE,
+	EQUAL_SPACING,		//use min/max + num bins to determine where 
+	CUSTOM
+}
+
+table UIHistogram {
+	type:UIHistogramType;
+	numbins:uint32;
+	binranges:FlatArray;	//Shape [2] for EQUAL_SPACING (min/max), or shape [2,numbins] for custom bin min/max values
+	y:FlatArray;			//Shape [numbins] - could be integer or floating point, positive or negative
+	binlabels:[string];		//Optional - used for discrete histograms (essentially, bar chart)	//TODO might want to register this value once + reuse?
+}
+
+table UISummaryStatistics {
+	bitmask:uint32;			//Bit mask that represents which of the primitives are actually present (FB doesn't support null primitives AFAIK)
+	min:FlatArray;			//Typed - but it would be more space efficient to use double...
+	max:FlatArray;
+	mean:double;
+	stdev:double;
+	countzero:long;
+	countpositive:long;
+	countnegative:long;
+	countnan:long;
+	countinf:long;
+}
+
+//Standard metrics related to current hardware status
+table UIHardwareState {
+	//TODO - do we want CPU/GPU utilization statistics and the like?
+	gpuMemory:[long];
+	hostMemory:long;
+}

libnd4j/include/graph/scheme/uigraphstatic.fbs

@@ -0,0 +1,77 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+include "utils.fbs";		//For: IntPair
+include "variable.fbs";		//For: VarType
+include "array.fbs";		//For: DataType
+
+namespace graph;
+
+enum UIInfoType:byte {
+	GRAPH_STRUCTURE,
+	SYTEM_INFO,
+	START_EVENTS		//An enum that means "we're done encoding static info, hereafter log contains only event pairs"
+}
+
+//Used only as a type/information header, so we know what to expect next (and hence how to decode it)
+table UIStaticInfoRecord {
+	infoType:UIInfoType;
+}
+
+table UISystemInfo {
+	//TODO - standard hardware information: CPU, GPU(s), total memory, etc.
+	// Also software info.
+	physicalCores:int;
+}
+
+
+//A representation of the graph structure for visual rendering
+table UIGraphStructure {
+    inputs:[string];
+    inputsPair:[IntPair];
+    outputs:[string];
+    variables:[UIVariable];
+    ops:[UIOp];
+}
+
+table UIVariable {
+    id:IntPair;                     //Existing IntPair class
+    name:string;
+    type:VarType;                   //Use existing VarType:     VARIABLE, CONSTANT, ARRAY, PLACEHOLDER
+	datatype:DType;
+	shape:[long];
+	controlDeps:[string];			//Input control dependencies: variable x -> this
+    outputOfOp:string;              //Null for placeholders/constants. For array type SDVariables, the name of the op it's an output of
+    inputsForOp:[string];           //Used as input for specific op. Mainly for quick lookup on front end
+    controlDepsForOp:[string];      //if a op control dependency (x -> opY) exists, then "opY" will be in this list
+    controlDepsForVar:[string];     //if a variable control dependency (x -> varY) exists, then "varY" will be in this list
+    gradientVariable:string;        //Variable (if any) corresponding to gradient of this variable
+	uiLabelExtra:string;			//Optional extra information (line(s)) to show in UI for variable label
+	constantValue:FlatArray;		//Array for the constant, usually for displaying scalars. May not be encoded for larger constants
+}
+
+table UIOp {
+	name:string;					//Unique name of this specific op
+	opName:string;					//Name of the op ("linspace", "add" etc)
+	inputs:[string];				//Name of input variables
+	outputs:[string];				//Name of output variables
+	controlDeps:[string];			//Name of control dependencies
+	uiLabelExtra:string;			//Optional extra information (line(s)) to show in UI for op label. Example: frame for enter/exit ops
+}

libnd4j/include/graph/scheme/utils.fbs

@@ -0,0 +1,88 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+namespace graph;
+
+table LongPair {
+    first:long; // first
+    second:long; // second
+}
+
+table LongTriple {
+    first:long;
+    second:long;
+    third:long;
+}
+
+table IntPair {
+    first:int;
+    second:int;
+}
+
+table IntTriple {
+    first:int;
+    second:int;
+    third:int;
+}
+
+enum OpType:byte {
+    TRANSFORM_FLOAT = 0,
+    TRANSFORM_SAME = 1,
+    TRANSFORM_BOOL = 2,
+    TRANSFORM_STRICT = 3,
+    TRANSFORM_ANY = 4,
+    REDUCE_FLOAT = 5, // for both reduce & reduce3
+    REDUCE_SAME = 6,
+    REDUCE_LONG = 7,
+    REDUCE_BOOL = 8,
+    INDEX_REDUCE = 9,
+    SCALAR = 10,
+    SCALAR_BOOL = 11,
+    BROADCAST = 12,
+    BROADCAST_BOOL = 13,
+    PAIRWISE = 14,
+    PAIRWISE_BOOL = 15,
+    REDUCE_3 = 16,
+    SUMMARYSTATS = 17,
+    SHAPE = 18,
+    AGGREGATION = 19, // ???
+    RANDOM = 20, //
+    CUSTOM = 21, // custom ops
+    GRAPH = 22, // another graph used as op
+    VARIABLE = 40,
+    BOOLEAN = 60, // booleanOps, for conditionals
+    LOGIC = 119, //
+    UDF = 120,
+}
+
+enum InputType:byte {
+    UNDEFINED,
+    NUMERIC,
+    STRINGULAR,
+    NUMERIC_SET,
+    STRINGULAR_SET,
+}
+
+enum OpClass:byte {
+    TRANSFORM = 0, // ops of this class return the same shape as primary input
+    REDUCTION = 1, // ops of this class return accumulation shape
+    MULTIPLICATOR = 2, // ops of this class may return whatever they want
+    GRAPH = 3, // embedded graph
+    CONDITIONAL = 4, // IF
+    LOOP = 5, // Various LOOPS
+}

libnd4j/include/graph/scheme/variable.fbs

@@ -0,0 +1,48 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+include "array.fbs";
+include "utils.fbs";
+
+namespace graph;
+
+// Variable type for variables
+enum VarType:byte {
+    VARIABLE,
+    CONSTANT,
+    ARRAY,
+    PLACEHOLDER
+}
+
+table FlatVariable {
+    id:IntPair; // ID of the Variable, in format of IntPair.first is node Id, IntPair.second is output index of the node
+    name:string; // symbolic ID of the Variable (if defined)
+    dtype:DType;
+
+    shape:[long]; // shape is absolutely optional. either shape or ndarray might be set
+    ndarray:FlatArray;
+
+    device:int; // default is -1, which means _auto_
+	variabletype:VarType;
+	
+	controlDeps:[string];
+	controlDepForOp:[string];
+	controlDepsForVar:[string];
+}
+
+root_type FlatVariable;