chore: import upstream snapshot with attribution

datavec/datavec-arrow/pom.xml

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ /* ******************************************************************************
+  ~  *
+  ~  *
+  ~  * This program and the accompanying materials are made available under the
+  ~  * terms of the Apache License, Version 2.0 which is available at
+  ~  * https://www.apache.org/licenses/LICENSE-2.0.
+  ~  *
+  ~  *  See the NOTICE file distributed with this work for additional
+  ~  *  information regarding copyright ownership.
+  ~  * Unless required by applicable law or agreed to in writing, software
+  ~  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  ~  * License for the specific language governing permissions and limitations
+  ~  * under the License.
+  ~  *
+  ~  * SPDX-License-Identifier: Apache-2.0
+  ~  ******************************************************************************/
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.eclipse.deeplearning4j</groupId>
+        <artifactId>datavec-parent</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>datavec-arrow</artifactId>
+
+    <name>datavec-arrow</name>
+
+    <properties>
+        <module.name>datavec.arrow</module.name>
+    </properties>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.moditect</groupId>
+                <artifactId>moditect-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>datavec-api</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-vector</artifactId>
+            <version>${arrow.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.code.findbugs</groupId>
+                    <artifactId>jsr305</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+    </dependencies>
+
+</project>

datavec/datavec-arrow/src/main/java/org/datavec/arrow/recordreader/ArrowRecord.java

@@ -0,0 +1,58 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.arrow.recordreader;
+
+import lombok.AllArgsConstructor;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataIndex;
+import org.datavec.api.writable.Writable;
+
+import java.net.URI;
+import java.util.List;
+
+@AllArgsConstructor
+public class ArrowRecord implements Record {
+    private ArrowWritableRecordBatch arrowWritableRecordBatch;
+    private  int index;
+    private URI recordUri;
+
+    @Override
+    public List<Writable> getRecord() {
+        return arrowWritableRecordBatch.get(index);
+    }
+
+    @Override
+    public void setRecord(List<Writable> record) {
+        arrowWritableRecordBatch.set(index,record);
+    }
+
+    @Override
+    public RecordMetaData getMetaData() {
+        RecordMetaData ret = new RecordMetaDataIndex(index,recordUri,ArrowRecordReader.class);
+        return ret;
+    }
+
+    @Override
+    public void setMetaData(RecordMetaData recordMetaData) {
+
+    }
+}

datavec/datavec-arrow/src/main/java/org/datavec/arrow/recordreader/ArrowRecordReader.java

@@ -0,0 +1,253 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.arrow.recordreader;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import lombok.val;
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.listener.RecordListener;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataIndex;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.split.FileSplit;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.transform.schema.Schema;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.*;
+
+import static org.datavec.arrow.ArrowConverter.readFromBytes;
+
+@Slf4j
+public class ArrowRecordReader implements RecordReader {
+
+    private InputSplit split;
+    private Configuration configuration;
+    private Iterator<String> pathsIter;
+    private int currIdx;
+    private String currentPath;
+    private Schema schema;
+    private List<Writable> recordAllocation = new ArrayList<>();
+    @Getter
+    private ArrowWritableRecordBatch currentBatch;
+    private List<RecordListener> recordListeners;
+
+    @Override
+    public void initialize(InputSplit split) {
+        this.split = split;
+        this.pathsIter = split.locationsPathIterator();
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) {
+        this.split = split;
+        this.pathsIter = split.locationsPathIterator();
+
+    }
+
+    @Override
+    public boolean batchesSupported() {
+        return true;
+    }
+
+    @Override
+    public List<List<Writable>> next(int num) {
+        if (currentBatch == null || currIdx >= currentBatch.size()) {
+            loadNextBatch();
+        }
+
+        if(num == currentBatch.getArrowRecordBatch().getLength()) {
+            currIdx += num;
+            return currentBatch;
+        }
+        else {
+            List<List<Writable>> ret = new ArrayList<>(num);
+            int numBatches = 0;
+            while(hasNext() && numBatches < num) {
+                ret.add(next());
+            }
+
+            return ret;
+        }
+
+
+    }
+
+    @Override
+    public List<Writable> next() {
+        if (currentBatch == null || currIdx >= currentBatch.size()) {
+            loadNextBatch();
+        }
+        else {
+            recordAllocation = currentBatch.get(currIdx++);
+        }
+
+        return recordAllocation;
+
+    }
+
+    private void loadNextBatch() {
+        String url = pathsIter.next();
+        try (InputStream inputStream = split.openInputStreamFor(url)) {
+            currIdx = 0;
+            byte[] arr = org.apache.commons.io.IOUtils.toByteArray(inputStream);
+            val read = readFromBytes(arr);
+            if(this.schema == null) {
+                this.schema = read.getFirst();
+            }
+
+            this.currentBatch = read.getRight();
+            this.recordAllocation = currentBatch.get(0);
+            currIdx++;
+            this.currentPath = url;
+        }catch(Exception e) {
+            log.error("",e);
+        }
+
+    }
+
+
+    @Override
+    public boolean hasNext() {
+        return pathsIter.hasNext() || currIdx < this.currentBatch.size();
+    }
+
+    @Override
+    public List<String> getLabels() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void reset() {
+        if(split != null) {
+            split.reset();
+        }
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Record nextRecord() {
+        next();
+        ArrowRecord ret =  new ArrowRecord(currentBatch,currIdx - 1,URI.create(currentPath));
+        return ret;
+    }
+
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) {
+        if(!(recordMetaData instanceof RecordMetaDataIndex)) {
+            throw new IllegalArgumentException("Unable to load from meta data. No index specified for record");
+        }
+
+        RecordMetaDataIndex index = (RecordMetaDataIndex) recordMetaData;
+        InputSplit fileSplit = new FileSplit(new File(index.getURI()));
+        initialize(fileSplit);
+        this.currIdx = (int) index.getIndex();
+        return nextRecord();
+    }
+
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) {
+        Map<String,List<RecordMetaData>> metaDataByUri = new HashMap<>();
+        //gather all unique locations for the metadata
+        //this will prevent initialization multiple times of the record
+        for(RecordMetaData recordMetaData : recordMetaDatas) {
+            if(!(recordMetaData instanceof RecordMetaDataIndex)) {
+                throw new IllegalArgumentException("Unable to load from meta data. No index specified for record");
+            }
+
+            List<RecordMetaData> recordMetaData1 = metaDataByUri.get(recordMetaData.getURI().toString());
+            if(recordMetaData1 == null) {
+                recordMetaData1 = new ArrayList<>();
+                metaDataByUri.put(recordMetaData.getURI().toString(),recordMetaData1);
+            }
+
+            recordMetaData1.add(recordMetaData);
+
+        }
+
+        List<Record> ret = new ArrayList<>();
+        for(String uri : metaDataByUri.keySet()) {
+            List<RecordMetaData> metaData = metaDataByUri.get(uri);
+            InputSplit fileSplit = new FileSplit(new File(URI.create(uri)));
+            initialize(fileSplit);
+            for(RecordMetaData index : metaData) {
+                RecordMetaDataIndex index2 = (RecordMetaDataIndex) index;
+                this.currIdx = (int) index2.getIndex();
+                ret.add(nextRecord());
+            }
+
+        }
+
+        return ret;
+    }
+
+    @Override
+    public List<RecordListener> getListeners() {
+        return recordListeners;
+    }
+
+    @Override
+    public void setListeners(RecordListener... listeners) {
+        this.recordListeners = new ArrayList<>(Arrays.asList(listeners));
+    }
+
+    @Override
+    public void setListeners(Collection<RecordListener> listeners) {
+        this.recordListeners = new ArrayList<>(listeners);
+    }
+
+    @Override
+    public void close() {
+        if(currentBatch != null) {
+            try {
+                currentBatch.close();
+            } catch (IOException e) {
+                log.error("",e);
+            }
+        }
+    }
+
+    @Override
+    public void setConf(Configuration conf) {
+        this.configuration = conf;
+    }
+
+    @Override
+    public Configuration getConf() {
+        return configuration;
+    }
+}

datavec/datavec-arrow/src/main/java/org/datavec/arrow/recordreader/ArrowRecordWriter.java

@@ -0,0 +1,102 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.arrow.recordreader;
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.writer.RecordWriter;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.split.partition.PartitionMetaData;
+import org.datavec.api.split.partition.Partitioner;
+import org.datavec.api.transform.schema.Schema;
+import org.datavec.api.writable.Writable;
+import org.datavec.arrow.ArrowConverter;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+public class ArrowRecordWriter implements RecordWriter {
+
+    private Configuration configuration;
+    private Schema schema;
+    private Partitioner partitioner;
+
+    public ArrowRecordWriter(Schema schema) {
+        this.schema = schema;
+    }
+
+    @Override
+    public boolean supportsBatch() {
+        return true;
+    }
+
+    @Override
+    public void initialize(InputSplit inputSplit, Partitioner partitioner) throws Exception {
+        this.partitioner = partitioner;
+        partitioner.init(inputSplit);
+
+    }
+
+    @Override
+    public void initialize(Configuration configuration, InputSplit split, Partitioner partitioner) throws Exception {
+        setConf(configuration);
+        this.partitioner = partitioner;
+    }
+
+    @Override
+    public PartitionMetaData write(List<Writable> record) throws IOException {
+        return writeBatch(Arrays.asList(record));
+    }
+
+    @Override
+    public PartitionMetaData writeBatch(List<List<Writable>> batch) throws IOException {
+        if(partitioner.needsNewPartition()) {
+            partitioner.currentOutputStream().flush();
+            partitioner.currentOutputStream().close();
+            partitioner.openNewStream();
+        }
+
+        if(batch instanceof ArrowWritableRecordBatch) {
+            ArrowWritableRecordBatch arrowWritableRecordBatch = (ArrowWritableRecordBatch) batch;
+            ArrowConverter.writeRecordBatchTo(arrowWritableRecordBatch,schema,partitioner.currentOutputStream());
+        }
+        else {
+            ArrowConverter.writeRecordBatchTo(batch, schema, partitioner.currentOutputStream());
+        }
+
+        partitioner.currentOutputStream().flush();
+        return PartitionMetaData.builder().numRecordsUpdated(batch.size()).build();
+    }
+
+    @Override
+    public void close() {
+    }
+
+    @Override
+    public void setConf(Configuration conf) {
+        this.configuration = conf;
+    }
+
+    @Override
+    public Configuration getConf() {
+        return configuration;
+    }
+}

datavec/datavec-arrow/src/main/java/org/datavec/arrow/recordreader/ArrowWritableRecordBatch.java

@@ -0,0 +1,309 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.arrow.recordreader;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+import org.datavec.api.transform.schema.Schema;
+import org.datavec.api.writable.NullWritable;
+import org.datavec.api.writable.Writable;
+import org.datavec.api.writable.batch.AbstractWritableRecordBatch;
+import org.datavec.arrow.ArrowConverter;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.*;
+
+/**
+ *
+ */
+@Data
+@AllArgsConstructor
+public class ArrowWritableRecordBatch extends AbstractWritableRecordBatch implements Closeable {
+
+    private List<FieldVector> list;
+    private int size;
+    private Schema schema;
+    private ArrowRecordBatch arrowRecordBatch;
+    private VectorSchemaRoot vectorLoader;
+    private VectorUnloader unloader;
+    private int offset,rows;
+
+    public ArrowWritableRecordBatch(List<FieldVector> list,Schema schema,int offset,int rows) {
+        this.list = list;
+        this.schema = schema;
+        //each column should have same number of rows
+        this.offset = offset;
+        this.size = rows;
+
+    }
+
+    /**
+     * An index in to an individual
+     * {@link ArrowRecordBatch}
+     * @param list the list of field vectors to use
+     * @param schema the schema to use
+     */
+    public ArrowWritableRecordBatch(List<FieldVector> list, Schema schema) {
+        this(list,schema,0,list.get(0).getValueCount());
+    }
+
+    @Override
+    public int size() {
+        return size;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size == 0;
+    }
+
+    @Override
+    public boolean contains(Object o) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Iterator<List<Writable>> iterator() {
+        return new ArrowListIterator();
+    }
+
+    @Override
+    public Object[] toArray() {
+        Object[] ret = new Object[size()];
+        for(int i = 0; i < ret.length; i++) {
+            ret[i] = get(i);
+        }
+
+        return ret;
+    }
+
+    @Override
+    public <T> T[] toArray(T[] ts) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean add(List<Writable> writable) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean remove(Object o) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean containsAll(Collection<?> collection) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean addAll(Collection<? extends List<Writable>> collection) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean addAll(int i,  Collection<? extends List<Writable>> collection) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean removeAll(Collection<?> collection) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean retainAll(Collection<?> collection) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void clear() {
+
+    }
+
+    @Override
+    public List<Writable> get(int i) {
+        List<Writable> ret = new ArrayList<>(schema.numColumns());
+        for(int column = 0; column < schema.numColumns(); column++) {
+            try {
+                if (!list.get(column).isNull(offset + i))
+                    ret.add(ArrowConverter.fromEntry(offset + i, list.get(column), schema.getType(column)));
+                else {
+                    ret.add(NullWritable.INSTANCE);
+                }
+            }catch (Exception e) {
+                ret.add(NullWritable.INSTANCE);
+
+            }
+        }
+        return ret;
+    }
+
+    @Override
+    public List<Writable> set(int i, List<Writable> writable) {
+        int rowOffset = offset + i;
+        List<Writable> old = get(i);
+        if(writable.size() != schema.numColumns()) {
+            throw new IllegalArgumentException("Unable to set value. Wrong input types coming in");
+        }
+
+        int colIdx = 0;
+        for(FieldVector fieldVector : list) {
+            ArrowConverter.setValue(schema.getType(colIdx),fieldVector,writable.get(colIdx),rowOffset);
+            colIdx++;
+        }
+
+        return old;
+    }
+
+    @Override
+    public void add(int i, List<Writable> writable) {
+        throw new UnsupportedOperationException();
+
+    }
+
+    @Override
+    public List<Writable> remove(int i) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int indexOf(Object o) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int lastIndexOf(Object o) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public ListIterator<List<Writable>> listIterator() {
+        return new ArrowListIterator();
+    }
+
+    @Override
+    public ListIterator<List<Writable>> listIterator(int i) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public List<List<Writable>> subList(int i, int i1) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+        ArrowWritableRecordBatch lists = (ArrowWritableRecordBatch) o;
+        return size == lists.size &&
+                Objects.equals(list, lists.list) &&
+                Objects.equals(schema, lists.schema);
+    }
+
+    @Override
+    public int hashCode() {
+
+        return Objects.hash(super.hashCode(), list, size, schema);
+    }
+
+    @Override
+    public void close() throws IOException {
+        if(arrowRecordBatch != null)
+            arrowRecordBatch.close();
+        if(vectorLoader != null)
+            vectorLoader.close();
+
+        list.forEach(ValueVector::close);
+    }
+
+
+    public List<List<Writable>> toArrayList() {
+        List<List<Writable>> ret = new ArrayList<>();
+        for(int i = 0; i < size(); i++) {
+            List<Writable> add = new ArrayList<>(get(i));
+            ret.add(add);
+        }
+
+        return ret;
+    }
+
+
+    private class ArrowListIterator implements ListIterator<List<Writable>> {
+        private int index;
+
+        @Override
+        public boolean hasNext() {
+            return index < size;
+        }
+
+        @Override
+        public List<Writable> next() {
+            return get(index++);
+        }
+
+        @Override
+        public boolean hasPrevious() {
+            return index > 0;
+        }
+
+        @Override
+        public List<Writable> previous() {
+            return get(index - 1);
+        }
+
+        @Override
+        public int nextIndex() {
+            return index + 1;
+        }
+
+        @Override
+        public int previousIndex() {
+            return index - 1;
+        }
+
+        @Override
+        public void remove() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void set(List<Writable> writables) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void add(List<Writable> writables) {
+            throw new UnsupportedOperationException();
+
+        }
+    }
+
+}

datavec/datavec-arrow/src/main/java/org/datavec/arrow/recordreader/ArrowWritableRecordTimeSeriesBatch.java

@@ -0,0 +1,291 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.arrow.recordreader;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+import org.datavec.api.transform.schema.Schema;
+import org.datavec.api.writable.Writable;
+import org.datavec.api.writable.batch.AbstractTimeSeriesWritableRecordBatch;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.*;
+
+/**
+ *
+ */
+@Data
+@AllArgsConstructor
+public class ArrowWritableRecordTimeSeriesBatch extends AbstractTimeSeriesWritableRecordBatch implements Closeable {
+
+    private List<FieldVector> list;
+    private int size;
+    private Schema schema;
+    private ArrowRecordBatch arrowRecordBatch;
+    private VectorSchemaRoot vectorLoader;
+    private VectorUnloader unloader;
+    private int timeSeriesStride;
+
+    /**
+     * An index in to an individual
+     * {@link ArrowRecordBatch}
+     * @param list the list of field vectors to use
+     * @param schema the schema to use
+     */
+    public ArrowWritableRecordTimeSeriesBatch(List<FieldVector> list, Schema schema,int timeSeriesStride) {
+        this.list = list;
+        this.schema = schema;
+        //each column should have same number of rows
+        this.timeSeriesStride = timeSeriesStride;
+        this.size = list.size() * list.get(0).getValueCount() / timeSeriesStride;
+
+    }
+
+    public List<List<List<Writable>>> toArrayList() {
+        List<List<List<Writable>>> ret = new ArrayList<>();
+        for(int i = 0; i < size(); i++) {
+            List<List<Writable>> timeStep = get(i);
+            List<List<Writable>> addTimeStep = new ArrayList<>();
+            for(int j = 0 ; j < timeStep.size(); j++) {
+                List<Writable> addingFrom = timeStep.get(j);
+                List<Writable> currRecord = new ArrayList<>(addingFrom);
+                addTimeStep.add(currRecord);
+            }
+
+            ret.add(addTimeStep);
+        }
+
+        return ret;
+    }
+
+
+    @Override
+    public int size() {
+        return size;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size == 0;
+    }
+
+    @Override
+    public boolean contains(Object o) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Iterator<List<List<Writable>>> iterator() {
+        return new ArrowListIterator();
+    }
+
+    @Override
+    public Object[] toArray() {
+        Object[] ret = new Object[size()];
+        for(int i = 0; i < ret.length; i++) {
+            ret[i] = get(i);
+        }
+        return ret;
+    }
+
+    @Override
+    public <T> T[] toArray(T[] ts) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean add(List<List<Writable>> writable) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean remove(Object o) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean containsAll(Collection<?> collection) {
+        return false;
+    }
+
+    @Override
+    public boolean addAll(Collection<? extends List<List<Writable>>> collection) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean addAll(int i,  Collection<? extends List<List<Writable>>> collection) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean removeAll(Collection<?> collection) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean retainAll(Collection<?> collection) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void clear() {
+
+    }
+
+    @Override
+    public List<List<Writable>> get(int i) {
+        ArrowWritableRecordBatch ret =  new ArrowWritableRecordBatch(list,schema,i ,timeSeriesStride / schema.numColumns());
+        return ret;
+    }
+
+    @Override
+    public List<List<Writable>> set(int i, List<List<Writable>> writable) {
+        ArrowWritableRecordBatch arrowWritableRecordBatch = (ArrowWritableRecordBatch) get(i);
+        for(int batch = 0; batch < writable.size(); batch++) {
+            arrowWritableRecordBatch.set(batch,writable.get(i));
+        }
+
+        return arrowWritableRecordBatch;
+    }
+
+    @Override
+    public void add(int i, List<List<Writable>> writable) {
+        throw new UnsupportedOperationException();
+
+    }
+
+    @Override
+    public List<List<Writable>> remove(int i) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int indexOf(Object o) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int lastIndexOf(Object o) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public ListIterator<List<List<Writable>>> listIterator() {
+        return new ArrowListIterator();
+    }
+
+    @Override
+    public ListIterator<List<List<Writable>>> listIterator(int i) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public List<List<List<Writable>>> subList(int i, int i1) {
+        return null;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+        ArrowWritableRecordTimeSeriesBatch lists = (ArrowWritableRecordTimeSeriesBatch) o;
+        return size == lists.size &&
+                Objects.equals(list, lists.list) &&
+                Objects.equals(schema, lists.schema);
+    }
+
+    @Override
+    public int hashCode() {
+
+        return Objects.hash(super.hashCode(), list, size, schema);
+    }
+
+    @Override
+    public void close() throws IOException {
+        if(arrowRecordBatch != null)
+            arrowRecordBatch.close();
+        if(vectorLoader != null)
+            vectorLoader.close();
+
+        list.forEach(ValueVector::close);
+    }
+
+
+    private class ArrowListIterator implements ListIterator<List<List<Writable>>> {
+        private int index;
+
+        @Override
+        public boolean hasNext() {
+            return index < size;
+        }
+
+        @Override
+        public List<List<Writable>> next() {
+            return get(index++);
+        }
+
+        @Override
+        public boolean hasPrevious() {
+            return index > 0;
+        }
+
+        @Override
+        public List<List<Writable>> previous() {
+            return get(index - 1);
+        }
+
+        @Override
+        public int nextIndex() {
+            return index + 1;
+        }
+
+        @Override
+        public int previousIndex() {
+            return index - 1;
+        }
+
+        @Override
+        public void remove() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void set(List<List<Writable>> writables) {
+            ArrowWritableRecordTimeSeriesBatch.this.set(index,writables);
+        }
+
+        @Override
+        public void add(List<List<Writable>> writables) {
+            throw new UnsupportedOperationException();
+
+        }
+    }
+
+
+}

datavec/datavec-arrow/src/main/java9/module-info.java

@@ -0,0 +1,12 @@
+open module datavec.arrow {
+    requires java.nio;
+    requires commons.io;
+    requires slf4j.api;
+    requires arrow.memory.core;
+    requires arrow.vector;
+    requires datavec.api;
+    requires nd4j.api;
+    requires nd4j.common;
+    exports org.datavec.arrow;
+    exports org.datavec.arrow.recordreader;
+}