chore: import upstream snapshot with attribution

datavec/datavec-api/pom.xml

@@ -0,0 +1,125 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ /* ******************************************************************************
+  ~  *
+  ~  *
+  ~  * This program and the accompanying materials are made available under the
+  ~  * terms of the Apache License, Version 2.0 which is available at
+  ~  * https://www.apache.org/licenses/LICENSE-2.0.
+  ~  *
+  ~  *  See the NOTICE file distributed with this work for additional
+  ~  *  information regarding copyright ownership.
+  ~  * Unless required by applicable law or agreed to in writing, software
+  ~  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  ~  * License for the specific language governing permissions and limitations
+  ~  * under the License.
+  ~  *
+  ~  * SPDX-License-Identifier: Apache-2.0
+  ~  ******************************************************************************/
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.eclipse.deeplearning4j</groupId>
+        <artifactId>datavec-parent</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>datavec-api</artifactId>
+
+    <properties>
+        <module.name>datavec.api</module.name>
+    </properties>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.moditect</groupId>
+                <artifactId>moditect-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.vintage</groupId>
+            <artifactId>junit-vintage-engine</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+            <version>${commons-codec.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>joda-time</groupId>
+            <artifactId>joda-time</artifactId>
+        </dependency>
+        <!-- ND4J Shaded Jackson Dependencies -->
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>jackson</artifactId>
+            <version>${nd4j.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.freemarker</groupId>
+            <artifactId>freemarker</artifactId>
+            <version>${freemarker.version}</version>
+        </dependency>
+        <!-- ND4J common, mainly for CompactHeapStringList -->
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>nd4j-common</artifactId>
+        </dependency>
+        <!-- ND4J api, mainly for NDArrayWritable and recognizing an INDArray in UnsafeWritableInjector -->
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>nd4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.clearspring.analytics</groupId>
+            <artifactId>stream</artifactId>
+            <version>${stream.analytics.version}</version>
+        </dependency>
+        <!-- csv parser, same dep used by spark -->
+        <dependency>
+            <groupId>com.opencsv</groupId>
+            <artifactId>opencsv</artifactId>
+            <version>${opencsv.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.tdunning</groupId>
+            <artifactId>t-digest</artifactId>
+            <version>${tdigest.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>it.unimi.dsi</groupId>
+            <artifactId>fastutil</artifactId>
+            <version>${fastutil.version}</version>
+        </dependency>
+
+    </dependencies>
+
+
+</project>

datavec/datavec-api/src/main/java/org/datavec/api/conf/Configurable.java

@@ -0,0 +1,31 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.conf;
+
+
+public interface Configurable {
+
+    /** Set the configuration to be used by this object. */
+    void setConf(Configuration conf);
+
+    /** Return the configuration used by this object. */
+    Configuration getConf();
+}

datavec/datavec-api/src/main/java/org/datavec/api/conf/Configured.java

@@ -0,0 +1,47 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.conf;
+
+public class Configured implements Configurable {
+
+    private Configuration conf;
+
+    /** Construct a Configured. */
+    public Configured() {
+        this(null);
+    }
+
+    /** Construct a Configured. */
+    public Configured(Configuration conf) {
+        setConf(conf);
+    }
+
+    // inherit javadoc
+    public void setConf(Configuration conf) {
+        this.conf = conf;
+    }
+
+    // inherit javadoc
+    public Configuration getConf() {
+        return conf;
+    }
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/exceptions/DataVecException.java

@@ -0,0 +1,43 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.exceptions;
+
+public class DataVecException extends Exception {
+    public DataVecException() {
+        super();
+    }
+
+    public DataVecException(String message) {
+        super(message);
+    }
+
+    public DataVecException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public DataVecException(Throwable cause) {
+        super(cause);
+    }
+
+    protected DataVecException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/exceptions/UnknownFormatException.java

@@ -0,0 +1,45 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.exceptions;
+
+/**
+ * @author sonali
+ */
+public class UnknownFormatException extends Exception {
+    public UnknownFormatException() {}
+
+    public UnknownFormatException(String message) {
+        super(message);
+    }
+
+    public UnknownFormatException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public UnknownFormatException(Throwable cause) {
+        super(cause);
+    }
+
+    public UnknownFormatException(String message, Throwable cause, boolean enableSuppression,
+                    boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/formats/input/BaseInputFormat.java

@@ -0,0 +1,80 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.formats.input;
+
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.WritableType;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * @author Adam Gibson
+ */
+public abstract class BaseInputFormat implements InputFormat {
+
+    @Override
+    public RecordReader createReader(InputSplit split) throws IOException, InterruptedException {
+        return createReader(split, null);
+    }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+
+    }
+
+    @Override
+    public void readFields(DataInput in) throws IOException {
+
+    }
+
+    @Override
+    public double toDouble() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public float toFloat() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int toInt() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long toLong() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void writeType(DataOutput out) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public WritableType getType(){
+        throw new UnsupportedOperationException();
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/formats/input/InputFormat.java

@@ -0,0 +1,47 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.formats.input;
+
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Writable;
+
+import java.io.IOException;
+
+public interface InputFormat extends Writable {
+
+    /**
+     * Creates a reader from an input split
+     * @param split the split to read
+     * @return the reader from the given input split
+     */
+    RecordReader createReader(InputSplit split, Configuration conf) throws IOException, InterruptedException;
+
+    /**
+     * Creates a reader from an input split
+     * @param split the split to read
+     * @return the reader from the given input split
+     */
+    RecordReader createReader(InputSplit split) throws IOException, InterruptedException;
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/CSVInputFormat.java

@@ -0,0 +1,57 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.formats.input.impl;
+
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.formats.input.BaseInputFormat;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.records.reader.impl.csv.CSVRecordReader;
+import org.datavec.api.split.InputSplit;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class CSVInputFormat extends BaseInputFormat {
+    @Override
+    public RecordReader createReader(InputSplit split, Configuration conf) throws IOException, InterruptedException {
+        return createReader(split);
+    }
+
+    @Override
+    public RecordReader createReader(InputSplit split) throws IOException, InterruptedException {
+        CSVRecordReader ret = new CSVRecordReader();
+        ret.initialize(split);
+        return ret;
+
+    }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+
+    }
+
+    @Override
+    public void readFields(DataInput in) throws IOException {
+
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/LibSvmInputFormat.java

@@ -0,0 +1,43 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.formats.input.impl;
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.formats.input.BaseInputFormat;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.records.reader.impl.misc.LibSvmRecordReader;
+import org.datavec.api.split.InputSplit;
+
+import java.io.IOException;
+
+/**
+ * Lib svm input format
+ *
+ * @author Adam Gibson
+ */
+public class LibSvmInputFormat extends BaseInputFormat {
+    @Override
+    public RecordReader createReader(InputSplit split, Configuration conf) throws IOException, InterruptedException {
+        RecordReader reader = new LibSvmRecordReader();
+        reader.initialize(conf, split);
+        return reader;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/LineInputFormat.java

@@ -0,0 +1,57 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.formats.input.impl;
+
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.formats.input.BaseInputFormat;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.records.reader.impl.LineRecordReader;
+import org.datavec.api.split.InputSplit;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class LineInputFormat extends BaseInputFormat {
+    @Override
+    public RecordReader createReader(InputSplit split, Configuration conf) throws IOException, InterruptedException {
+        return createReader(split);
+    }
+
+    @Override
+    public RecordReader createReader(InputSplit split) throws IOException, InterruptedException {
+        LineRecordReader ret = new LineRecordReader();
+        ret.initialize(split);
+        return ret;
+
+    }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+
+    }
+
+    @Override
+    public void readFields(DataInput in) throws IOException {
+
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/ListStringInputFormat.java

@@ -0,0 +1,128 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.formats.input.impl;
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.formats.input.InputFormat;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.records.reader.impl.collection.ListStringRecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.WritableType;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class ListStringInputFormat implements InputFormat {
+    /**
+     * Creates a reader from an input split
+     *
+     * @param split the split to read
+     * @param conf
+     * @return the reader from the given input split
+     */
+    @Override
+    public RecordReader createReader(InputSplit split, Configuration conf) throws IOException, InterruptedException {
+        RecordReader reader = new ListStringRecordReader();
+        reader.initialize(conf, split);
+        return reader;
+    }
+
+    /**
+     * Creates a reader from an input split
+     *
+     * @param split the split to read
+     * @return the reader from the given input split
+     */
+    @Override
+    public RecordReader createReader(InputSplit split) throws IOException, InterruptedException {
+        RecordReader reader = new ListStringRecordReader();
+        reader.initialize(split);
+        return reader;
+    }
+
+    /**
+     * Serialize the fields of this object to <code>out</code>.
+     *
+     * @param out <code>DataOuput</code> to serialize this object into.
+     * @throws IOException
+     */
+    @Override
+    public void write(DataOutput out) throws IOException {
+
+    }
+
+    /**
+     * Deserialize the fields of this object from <code>in</code>.
+     * <p>
+     * <p>For efficiency, implementations should attempt to re-use storage in the
+     * existing object where possible.</p>
+     *
+     * @param in <code>DataInput</code> to deseriablize this object from.
+     * @throws IOException
+     */
+    @Override
+    public void readFields(DataInput in) throws IOException {
+
+    }
+
+    /**
+     * Convert Writable to double. Whether this is supported depends on the specific writable.
+     */
+    @Override
+    public double toDouble() {
+        return 0;
+    }
+
+    /**
+     * Convert Writable to float. Whether this is supported depends on the specific writable.
+     */
+    @Override
+    public float toFloat() {
+        return 0;
+    }
+
+    /**
+     * Convert Writable to int. Whether this is supported depends on the specific writable.
+     */
+    @Override
+    public int toInt() {
+        return 0;
+    }
+
+    /**
+     * Convert Writable to long. Whether this is supported depends on the specific writable.
+     */
+    @Override
+    public long toLong() {
+        return 0;
+    }
+
+    @Override
+    public WritableType getType() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void writeType(DataOutput out) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/MatlabInputFormat.java

@@ -0,0 +1,48 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.formats.input.impl;
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.formats.input.BaseInputFormat;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.records.reader.impl.misc.MatlabRecordReader;
+import org.datavec.api.split.InputSplit;
+
+import java.io.IOException;
+
+/**
+ * Matlab input format
+ *
+ * @author Adam Gibson
+ */
+public class MatlabInputFormat extends BaseInputFormat {
+    @Override
+    public RecordReader createReader(InputSplit split, Configuration conf) throws IOException, InterruptedException {
+        return createReader(split);
+    }
+
+    @Override
+    public RecordReader createReader(InputSplit split) throws IOException, InterruptedException {
+        RecordReader reader = new MatlabRecordReader();
+        reader.initialize(split);
+        return reader;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/SVMLightInputFormat.java

@@ -0,0 +1,52 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.formats.input.impl;
+
+
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.formats.input.BaseInputFormat;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.records.reader.impl.misc.SVMLightRecordReader;
+import org.datavec.api.split.InputSplit;
+
+import java.io.IOException;
+
+/**
+ * SVMLight input format
+ *
+ * @author Adam Gibson
+ */
+public class SVMLightInputFormat extends BaseInputFormat {
+    @Override
+    public RecordReader createReader(InputSplit split, Configuration conf) throws IOException, InterruptedException {
+        return createReader(split);
+    }
+
+    @Override
+    public RecordReader createReader(InputSplit split) throws IOException, InterruptedException {
+        SVMLightRecordReader reader = new SVMLightRecordReader();
+        reader.initialize(split);
+        return reader;
+    }
+
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/formats/output/OutputFormat.java

@@ -0,0 +1,38 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.formats.output;
+
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.exceptions.DataVecException;
+import org.datavec.api.records.writer.RecordWriter;
+
+public interface OutputFormat {
+
+    public static final String OUTPUT_PATH = "org.nd4j.outputpath";
+
+    /**
+     * Create a record writer
+     * @return the created writer
+     */
+    RecordWriter createWriter(Configuration conf) throws DataVecException;
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/formats/output/impl/CSVOutputFormat.java

@@ -0,0 +1,36 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.formats.output.impl;
+
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.exceptions.DataVecException;
+import org.datavec.api.formats.output.OutputFormat;
+import org.datavec.api.records.writer.RecordWriter;
+import org.datavec.api.records.writer.impl.csv.CSVRecordWriter;
+
+public class CSVOutputFormat implements OutputFormat {
+    @Override
+    public RecordWriter createWriter(Configuration conf) throws DataVecException {
+        String outputPath = conf.get(OutputFormat.OUTPUT_PATH, ".");
+        return new CSVRecordWriter();
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/formats/output/impl/LibSvmOutputFormat.java

@@ -0,0 +1,39 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.formats.output.impl;
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.exceptions.DataVecException;
+import org.datavec.api.formats.output.OutputFormat;
+import org.datavec.api.records.writer.RecordWriter;
+import org.datavec.api.records.writer.impl.misc.LibSvmRecordWriter;
+
+/**
+ * @author Adam Gibson
+ */
+public class LibSvmOutputFormat implements OutputFormat {
+    @Override
+    public RecordWriter createWriter(Configuration conf) throws DataVecException {
+        RecordWriter writer = new LibSvmRecordWriter();
+        writer.setConf(conf);
+        return writer;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/formats/output/impl/LineOutputFormat.java

@@ -0,0 +1,40 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.formats.output.impl;
+
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.exceptions.DataVecException;
+import org.datavec.api.formats.output.OutputFormat;
+import org.datavec.api.records.writer.RecordWriter;
+import org.datavec.api.records.writer.impl.LineRecordWriter;
+
+/**
+ * Line output format
+ * @author Adam Gibson
+ */
+public class LineOutputFormat implements OutputFormat {
+    @Override
+    public RecordWriter createWriter(Configuration conf) throws DataVecException {
+        String outputPath = conf.get(OutputFormat.OUTPUT_PATH, ".");
+        return new LineRecordWriter();
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/formats/output/impl/SVMLightOutputFormat.java

@@ -0,0 +1,42 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.formats.output.impl;
+
+
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.exceptions.DataVecException;
+import org.datavec.api.formats.output.OutputFormat;
+import org.datavec.api.records.writer.RecordWriter;
+import org.datavec.api.records.writer.impl.misc.SVMLightRecordWriter;
+
+public class SVMLightOutputFormat implements OutputFormat {
+    @Override
+    public RecordWriter createWriter(Configuration conf) throws DataVecException {
+        String outputPath = conf.get(OutputFormat.OUTPUT_PATH, ".");
+        try {
+            //return new LineRecordWriter(new File(outputPath));
+            return new SVMLightRecordWriter();
+        } catch (Exception e) {
+            throw new DataVecException(e);
+        }
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/BinaryComparable.java

@@ -0,0 +1,72 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io;
+
+public abstract class BinaryComparable implements Comparable<BinaryComparable> {
+
+    /**
+     * Return n st bytes 0..n-1 from {#getBytes()} are valid.
+     */
+    public abstract int getLength();
+
+    /**
+     * Return representative byte array for this instance.
+     */
+    public abstract byte[] getBytes();
+
+    /**
+     * Compare bytes from {#getBytes()}.
+     * @see org.apache.hadoop.io.WritableComparator#compareBytes(byte[],int,int,byte[],int,int)
+     */
+    public int compareTo(BinaryComparable other) {
+        if (this == other)
+            return 0;
+        return WritableComparator.compareBytes(getBytes(), 0, getLength(), other.getBytes(), 0, other.getLength());
+    }
+
+    /**
+     * Compare bytes from {#getBytes()} to those provided.
+     */
+    public int compareTo(byte[] other, int off, int len) {
+        return WritableComparator.compareBytes(getBytes(), 0, getLength(), other, off, len);
+    }
+
+    /**
+     * Return true if bytes from {#getBytes()} match.
+     */
+    public boolean equals(Object other) {
+        if (!(other instanceof BinaryComparable))
+            return false;
+        BinaryComparable that = (BinaryComparable) other;
+        if (this.getLength() != that.getLength())
+            return false;
+        return this.compareTo(that) == 0;
+    }
+
+    /**
+     * Return a hash of the bytes returned from {#getBytes()}.
+     * @see org.apache.hadoop.io.WritableComparator#hashBytes(byte[],int)
+     */
+    public int hashCode() {
+        return WritableComparator.hashBytes(getBytes(), getLength());
+    }
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/DataInputBuffer.java

@@ -0,0 +1,89 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+
+public class DataInputBuffer extends DataInputStream {
+    private static class Buffer extends ByteArrayInputStream {
+        public Buffer() {
+            super(new byte[] {});
+        }
+
+        public void reset(byte[] input, int start, int length) {
+            this.buf = input;
+            this.count = start + length;
+            this.mark = start;
+            this.pos = start;
+        }
+
+        public byte[] getData() {
+            return buf;
+        }
+
+        public int getPosition() {
+            return pos;
+        }
+
+        public int getLength() {
+            return count;
+        }
+    }
+
+    private Buffer buffer;
+
+    /** Constructs a new empty buffer. */
+    public DataInputBuffer() {
+        this(new Buffer());
+    }
+
+    private DataInputBuffer(Buffer buffer) {
+        super(buffer);
+        this.buffer = buffer;
+    }
+
+    /** Resets the data that the buffer reads. */
+    public void reset(byte[] input, int length) {
+        buffer.reset(input, 0, length);
+    }
+
+    /** Resets the data that the buffer reads. */
+    public void reset(byte[] input, int start, int length) {
+        buffer.reset(input, start, length);
+    }
+
+    public byte[] getData() {
+        return buffer.getData();
+    }
+
+    /** Returns the current position in the input. */
+    public int getPosition() {
+        return buffer.getPosition();
+    }
+
+    /** Returns the length of the input. */
+    public int getLength() {
+        return buffer.getLength();
+    }
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/DataOutputBuffer.java

@@ -0,0 +1,100 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io;
+
+import java.io.*;
+
+public class DataOutputBuffer extends DataOutputStream {
+
+    private static class Buffer extends ByteArrayOutputStream {
+        public byte[] getData() {
+            return buf;
+        }
+
+        public int getLength() {
+            return count;
+        }
+
+        public Buffer() {
+            super();
+        }
+
+        public Buffer(int size) {
+            super(size);
+        }
+
+        public void write(DataInput in, int len) throws IOException {
+            int newcount = count + len;
+            if (newcount > buf.length) {
+                byte newbuf[] = new byte[Math.max(buf.length << 1, newcount)];
+                System.arraycopy(buf, 0, newbuf, 0, count);
+                buf = newbuf;
+            }
+            in.readFully(buf, count, len);
+            count = newcount;
+        }
+    }
+
+    private Buffer buffer;
+
+    /** Constructs a new empty buffer. */
+    public DataOutputBuffer() {
+        this(new Buffer());
+    }
+
+    public DataOutputBuffer(int size) {
+        this(new Buffer(size));
+    }
+
+    private DataOutputBuffer(Buffer buffer) {
+        super(buffer);
+        this.buffer = buffer;
+    }
+
+    /** Returns the current contents of the buffer.
+     *  Data is only valid to {@link #getLength()}.
+     */
+    public byte[] getData() {
+        return buffer.getData();
+    }
+
+    /** Returns the length of the valid data currently in the buffer. */
+    public int getLength() {
+        return buffer.getLength();
+    }
+
+    /** Resets the buffer to empty. */
+    public DataOutputBuffer reset() {
+        this.written = 0;
+        buffer.reset();
+        return this;
+    }
+
+    /** Writes bytes from a DataInput directly into the buffer. */
+    public void write(DataInput in, int length) throws IOException {
+        buffer.write(in, length);
+    }
+
+    /** Write to a file stream */
+    public void writeTo(OutputStream out) throws IOException {
+        buffer.writeTo(out);
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/RawComparator.java

@@ -0,0 +1,30 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io;
+
+import java.util.Comparator;
+
+
+public interface RawComparator<T> extends Comparator<T> {
+
+    public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2);
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/WritableComparable.java

@@ -0,0 +1,27 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io;
+
+
+import org.datavec.api.writable.Writable;
+
+public interface WritableComparable<T> extends Writable, Comparable<T> {
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/WritableComparator.java

@@ -0,0 +1,230 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io;
+
+
+import org.datavec.api.util.ReflectionUtils;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.util.HashMap;
+
+
+public class WritableComparator implements RawComparator {
+
+    private static HashMap<Class, WritableComparator> comparators = new HashMap<>(); // registry
+
+    /** Get a comparator for a {@link WritableComparable} implementation. */
+    public static synchronized WritableComparator get(Class<? extends WritableComparable> c) {
+        WritableComparator comparator = comparators.get(c);
+        if (comparator == null) {
+            // force the static initializers to run
+            forceInit(c);
+            // look to see if it is defined now
+            comparator = comparators.get(c);
+            // if not, use the generic one
+            if (comparator == null) {
+                comparator = new WritableComparator(c, true);
+                comparators.put(c, comparator);
+            }
+        }
+        return comparator;
+    }
+
+    /**
+     * Force initialization of the static members.
+     * As of Java 5, referencing a class doesn't force it to initialize. Since
+     * this class requires that the classes be initialized to declare their
+     * comparators, we force that initialization to happen.
+     * @param cls the class to initialize
+     */
+    private static void forceInit(Class<?> cls) {
+        try {
+            Class.forName(cls.getName(), true, cls.getClassLoader());
+        } catch (ClassNotFoundException e) {
+            throw new IllegalArgumentException("Can't initialize class " + cls, e);
+        }
+    }
+
+    /** Register an optimized comparator for a {@link WritableComparable}
+     * implementation. */
+    public static synchronized void define(Class c, WritableComparator comparator) {
+        comparators.put(c, comparator);
+    }
+
+
+    private final Class<? extends WritableComparable> keyClass;
+    private final WritableComparable key1;
+    private final WritableComparable key2;
+    private final DataInputBuffer buffer;
+
+    /** Construct for a {@link WritableComparable} implementation. */
+    protected WritableComparator(Class<? extends WritableComparable> keyClass) {
+        this(keyClass, false);
+    }
+
+    protected WritableComparator(Class<? extends WritableComparable> keyClass, boolean createInstances) {
+        this.keyClass = keyClass;
+        if (createInstances) {
+            key1 = newKey();
+            key2 = newKey();
+            buffer = new DataInputBuffer();
+        } else {
+            key1 = key2 = null;
+            buffer = null;
+        }
+    }
+
+    /** Returns the WritableComparable implementation class. */
+    public Class<? extends WritableComparable> getKeyClass() {
+        return keyClass;
+    }
+
+    /** Construct a new {@link WritableComparable} instance. */
+    public WritableComparable newKey() {
+        return ReflectionUtils.newInstance(keyClass, null);
+    }
+
+    /** Optimization hook.  Override this to make SequenceFile.Sorter's scream.
+     *
+     * <p>The default implementation reads the data into two {@link
+     * WritableComparable}s (using {@link
+     * Writable#readFields(DataInput)}, then calls {@link
+     * #compare(WritableComparable,WritableComparable)}.
+     */
+    public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+        try {
+            buffer.reset(b1, s1, l1); // parse key1
+            key1.readFields(buffer);
+
+            buffer.reset(b2, s2, l2); // parse key2
+            key2.readFields(buffer);
+
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+        return compare(key1, key2); // compare them
+    }
+
+    /** Compare two WritableComparables.
+     *
+     * <p> The default implementation uses the natural ordering, calling {@link
+     * Comparable#compareTo(Object)}. */
+    @SuppressWarnings("unchecked")
+    public int compare(WritableComparable a, WritableComparable b) {
+        return a.compareTo(b);
+    }
+
+    public int compare(Object a, Object b) {
+        return compare((WritableComparable) a, (WritableComparable) b);
+    }
+
+    /** Lexicographic order of binary data. */
+    public static int compareBytes(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+        int end1 = s1 + l1;
+        int end2 = s2 + l2;
+        for (int i = s1, j = s2; i < end1 && j < end2; i++, j++) {
+            int a = (b1[i] & 0xff);
+            int b = (b2[j] & 0xff);
+            if (a != b) {
+                return a - b;
+            }
+        }
+        return l1 - l2;
+    }
+
+    /** Compute hash for binary data. */
+    public static int hashBytes(byte[] bytes, int offset, int length) {
+        int hash = 1;
+        for (int i = offset; i < offset + length; i++)
+            hash = (31 * hash) + (int) bytes[i];
+        return hash;
+    }
+
+    /** Compute hash for binary data. */
+    public static int hashBytes(byte[] bytes, int length) {
+        return hashBytes(bytes, 0, length);
+    }
+
+    /** Parse an unsigned short from a byte array. */
+    public static int readUnsignedShort(byte[] bytes, int start) {
+        return (((bytes[start] & 0xff) << 8) + ((bytes[start + 1] & 0xff)));
+    }
+
+    /** Parse an integer from a byte array. */
+    public static int readInt(byte[] bytes, int start) {
+        return (((bytes[start] & 0xff) << 24) + ((bytes[start + 1] & 0xff) << 16) + ((bytes[start + 2] & 0xff) << 8)
+                        + ((bytes[start + 3] & 0xff)));
+
+    }
+
+    /** Parse a float from a byte array. */
+    public static float readFloat(byte[] bytes, int start) {
+        return Float.intBitsToFloat(readInt(bytes, start));
+    }
+
+    /** Parse a long from a byte array. */
+    public static long readLong(byte[] bytes, int start) {
+        return ((long) (readInt(bytes, start)) << 32) + (readInt(bytes, start + 4) & 0xFFFFFFFFL);
+    }
+
+    /** Parse a double from a byte array. */
+    public static double readDouble(byte[] bytes, int start) {
+        return Double.longBitsToDouble(readLong(bytes, start));
+    }
+
+    /**
+     * Reads a zero-compressed encoded long from a byte array and returns it.
+     * @param bytes byte array with decode long
+     * @param start starting index
+     * @throws java.io.IOException
+     * @return deserialized long
+     */
+    public static long readVLong(byte[] bytes, int start) throws IOException {
+        int len = bytes[start];
+        if (len >= -112) {
+            return len;
+        }
+        boolean isNegative = (len < -120);
+        len = isNegative ? -(len + 120) : -(len + 112);
+        if (start + 1 + len > bytes.length)
+            throw new IOException("Not enough number of bytes for a zero-compressed integer");
+        long i = 0;
+        for (int idx = 0; idx < len; idx++) {
+            i = i << 8;
+            i = i | (bytes[start + 1 + idx] & 0xFF);
+        }
+        return (isNegative ? (~i) : i);
+    }
+
+    /**
+     * Reads a zero-compressed encoded integer from a byte array and returns it.
+     * @param bytes byte array with the encoded integer
+     * @param start start index
+     * @throws java.io.IOException
+     * @return deserialized integer
+     */
+    public static int readVInt(byte[] bytes, int start) throws IOException {
+        return (int) readVLong(bytes, start);
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/WritableConverter.java

@@ -0,0 +1,36 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io;
+
+import org.datavec.api.io.converters.WritableConverterException;
+import org.datavec.api.writable.Writable;
+
+public interface WritableConverter {
+
+
+    /**
+     * Convert a writable to another kind of writable
+     * @param writable the writable to convert
+     * @return the converted writable
+     */
+    Writable convert(Writable writable) throws WritableConverterException;
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/WritableUtils.java

@@ -0,0 +1,421 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io;
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.util.ReflectionUtils;
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+public final class WritableUtils {
+
+
+
+    public static byte[] readCompressedByteArray(DataInput in) throws IOException {
+        int length = in.readInt();
+        if (length == -1)
+            return null;
+        byte[] buffer = new byte[length];
+        in.readFully(buffer); // could/should use readFully(buffer,0,length)?
+        GZIPInputStream gzi = new GZIPInputStream(new ByteArrayInputStream(buffer, 0, buffer.length));
+        byte[] outbuf = new byte[length];
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        int len;
+        while ((len = gzi.read(outbuf, 0, outbuf.length)) != -1) {
+            bos.write(outbuf, 0, len);
+        }
+        byte[] decompressed = bos.toByteArray();
+        bos.close();
+        gzi.close();
+        return decompressed;
+    }
+
+    public static void skipCompressedByteArray(DataInput in) throws IOException {
+        int length = in.readInt();
+        if (length != -1) {
+            skipFully(in, length);
+        }
+    }
+
+    public static int writeCompressedByteArray(DataOutput out, byte[] bytes) throws IOException {
+        if (bytes != null) {
+            ByteArrayOutputStream bos = new ByteArrayOutputStream();
+            GZIPOutputStream gzout = new GZIPOutputStream(bos);
+            gzout.write(bytes, 0, bytes.length);
+            gzout.close();
+            byte[] buffer = bos.toByteArray();
+            int len = buffer.length;
+            out.writeInt(len);
+            out.write(buffer, 0, len);
+            /* debug only! Once we have confidence, can lose this. */
+            return ((bytes.length != 0) ? (100 * buffer.length) / bytes.length : 0);
+        } else {
+            out.writeInt(-1);
+            return -1;
+        }
+    }
+
+
+    /* Ugly utility, maybe someone else can do this better  */
+    public static String readCompressedString(DataInput in) throws IOException {
+        byte[] bytes = readCompressedByteArray(in);
+        if (bytes == null)
+            return null;
+        return new String(bytes, StandardCharsets.UTF_8);
+    }
+
+
+    public static int writeCompressedString(DataOutput out, String s) throws IOException {
+        return writeCompressedByteArray(out, (s != null) ? s.getBytes(StandardCharsets.UTF_8) : null);
+    }
+
+    /*
+     *
+     * Write a String as a Network Int n, followed by n Bytes
+     * Alternative to 16 bit read/writeUTF.
+     * Encoding standard is... ?
+     *
+     */
+    public static void writeString(DataOutput out, String s) throws IOException {
+        if (s != null) {
+            byte[] buffer = s.getBytes(StandardCharsets.UTF_8);
+            int len = buffer.length;
+            out.writeInt(len);
+            out.write(buffer, 0, len);
+        } else {
+            out.writeInt(-1);
+        }
+    }
+
+    /*
+     * Read a String as a Network Int n, followed by n Bytes
+     * Alternative to 16 bit read/writeUTF.
+     * Encoding standard is... ?
+     *
+     */
+    public static String readString(DataInput in) throws IOException {
+        int length = in.readInt();
+        if (length == -1)
+            return null;
+        byte[] buffer = new byte[length];
+        in.readFully(buffer); // could/should use readFully(buffer,0,length)?
+        return new String(buffer, StandardCharsets.UTF_8);
+    }
+
+
+    /*
+     * Write a String array as a Nework Int N, followed by Int N Byte Array Strings.
+     * Could be generalised using introspection.
+     *
+     */
+    public static void writeStringArray(DataOutput out, String[] s) throws IOException {
+        out.writeInt(s.length);
+        for (int i = 0; i < s.length; i++) {
+            writeString(out, s[i]);
+        }
+    }
+
+    /*
+     * Write a String array as a Nework Int N, followed by Int N Byte Array of
+     * compressed Strings. Handles also null arrays and null values.
+     * Could be generalised using introspection.
+     *
+     */
+    public static void writeCompressedStringArray(DataOutput out, String[] s) throws IOException {
+        if (s == null) {
+            out.writeInt(-1);
+            return;
+        }
+        out.writeInt(s.length);
+        for (int i = 0; i < s.length; i++) {
+            writeCompressedString(out, s[i]);
+        }
+    }
+
+    /*
+     * Write a String array as a Nework Int N, followed by Int N Byte Array Strings.
+     * Could be generalised using introspection. Actually this bit couldn't...
+     *
+     */
+    public static String[] readStringArray(DataInput in) throws IOException {
+        int len = in.readInt();
+        if (len == -1)
+            return null;
+        String[] s = new String[len];
+        for (int i = 0; i < len; i++) {
+            s[i] = readString(in);
+        }
+        return s;
+    }
+
+
+    /*
+     * Write a String array as a Nework Int N, followed by Int N Byte Array Strings.
+     * Could be generalised using introspection. Handles null arrays and null values.
+     *
+     */
+    public static String[] readCompressedStringArray(DataInput in) throws IOException {
+        int len = in.readInt();
+        if (len == -1)
+            return null;
+        String[] s = new String[len];
+        for (int i = 0; i < len; i++) {
+            s[i] = readCompressedString(in);
+        }
+        return s;
+    }
+
+
+    /*
+     *
+     * Test Utility Method Display Byte Array.
+     *
+     */
+    public static void displayByteArray(byte[] record) {
+        int i;
+        for (i = 0; i < record.length - 1; i++) {
+            if (i % 16 == 0) {
+                System.out.println();
+            }
+            System.out.print(Integer.toHexString(record[i] >> 4 & 0x0F));
+            System.out.print(Integer.toHexString(record[i] & 0x0F));
+            System.out.print(",");
+        }
+        System.out.print(Integer.toHexString(record[i] >> 4 & 0x0F));
+        System.out.print(Integer.toHexString(record[i] & 0x0F));
+        System.out.println();
+    }
+
+    /**
+     * Make a copy of a writable object using serialization to a buffer.
+     * @param orig The object to copy
+     * @return The copied object
+     */
+    public static <T extends Writable> T clone(T orig, Configuration conf) {
+        try {
+            @SuppressWarnings("unchecked") // Unchecked cast from Class to Class<T>
+            T newInst = ReflectionUtils.newInstance((Class<T>) orig.getClass(), conf);
+            ReflectionUtils.copy(conf, orig, newInst);
+            return newInst;
+        } catch (IOException e) {
+            throw new RuntimeException("Error writing/reading clone buffer", e);
+        }
+    }
+
+
+
+    /**
+     * Serializes an integer to a binary stream with zero-compressed encoding.
+     * For -120 <= i <= 127, only one byte is used with the actual value.
+     * For other values of i, the first byte value indicates whether the
+     * integer is positive or negative, and the number of bytes that follow.
+     * If the first byte value v is between -121 and -124, the following integer
+     * is positive, with number of bytes that follow are -(v+120).
+     * If the first byte value v is between -125 and -128, the following integer
+     * is negative, with number of bytes that follow are -(v+124). Bytes are
+     * stored in the high-non-zero-byte-first order.
+     *
+     * @param stream Binary output stream
+     * @param i Integer to be serialized
+     * @throws java.io.IOException
+     */
+    public static void writeVInt(DataOutput stream, int i) throws IOException {
+        writeVLong(stream, i);
+    }
+
+    /**
+     * Serializes a long to a binary stream with zero-compressed encoding.
+     * For -112 <= i <= 127, only one byte is used with the actual value.
+     * For other values of i, the first byte value indicates whether the
+     * long is positive or negative, and the number of bytes that follow.
+     * If the first byte value v is between -113 and -120, the following long
+     * is positive, with number of bytes that follow are -(v+112).
+     * If the first byte value v is between -121 and -128, the following long
+     * is negative, with number of bytes that follow are -(v+120). Bytes are
+     * stored in the high-non-zero-byte-first order.
+     *
+     * @param stream Binary output stream
+     * @param i Long to be serialized
+     * @throws java.io.IOException
+     */
+    public static void writeVLong(DataOutput stream, long i) throws IOException {
+        if (i >= -112 && i <= 127) {
+            stream.writeByte((byte) i);
+            return;
+        }
+
+        int len = -112;
+        if (i < 0) {
+            i ^= -1L; // take one's complement'
+            len = -120;
+        }
+
+        long tmp = i;
+        while (tmp != 0) {
+            tmp = tmp >> 8;
+            len--;
+        }
+
+        stream.writeByte((byte) len);
+
+        len = (len < -120) ? -(len + 120) : -(len + 112);
+
+        for (int idx = len; idx != 0; idx--) {
+            int shiftbits = (idx - 1) * 8;
+            long mask = 0xFFL << shiftbits;
+            stream.writeByte((byte) ((i & mask) >> shiftbits));
+        }
+    }
+
+
+    /**
+     * Reads a zero-compressed encoded long from input stream and returns it.
+     * @param stream Binary input stream
+     * @throws java.io.IOException
+     * @return deserialized long from stream.
+     */
+    public static long readVLong(DataInput stream) throws IOException {
+        byte firstByte = stream.readByte();
+        int len = decodeVIntSize(firstByte);
+        if (len == 1) {
+            return firstByte;
+        }
+        long i = 0;
+        for (int idx = 0; idx < len - 1; idx++) {
+            byte b = stream.readByte();
+            i = i << 8;
+            i = i | (b & 0xFF);
+        }
+        return (isNegativeVInt(firstByte) ? (i ^ -1L) : i);
+    }
+
+    /**
+     * Reads a zero-compressed encoded integer from input stream and returns it.
+     * @param stream Binary input stream
+     * @throws java.io.IOException
+     * @return deserialized integer from stream.
+     */
+    public static int readVInt(DataInput stream) throws IOException {
+        return (int) readVLong(stream);
+    }
+
+    /**
+     * Given the first byte of a vint/vlong, determine the sign
+     * @param value the first byte
+     * @return is the value negative
+     */
+    public static boolean isNegativeVInt(byte value) {
+        return value < -120 || (value >= -112 && value < 0);
+    }
+
+    /**
+     * Parse the first byte of a vint/vlong to determine the number of bytes
+     * @param value the first byte of the vint/vlong
+     * @return the total number of bytes (1 to 9)
+     */
+    public static int decodeVIntSize(byte value) {
+        if (value >= -112) {
+            return 1;
+        } else if (value < -120) {
+            return -119 - value;
+        }
+        return -111 - value;
+    }
+
+    /**
+     * Get the encoded length if an integer is stored in a variable-length format
+     * @return the encoded length
+     */
+    public static int getVIntSize(long i) {
+        if (i >= -112 && i <= 127) {
+            return 1;
+        }
+
+        if (i < 0) {
+            i ^= -1L; // take one's complement'
+        }
+        // find the number of bytes with non-leading zeros
+        int dataBits = Long.SIZE - Long.numberOfLeadingZeros(i);
+        // find the number of data bytes + length byte
+        return (dataBits + 7) / 8 + 1;
+    }
+
+    /**
+     * Read an Enum value from DataInput, Enums are read and written
+     * using String values.
+     * @param <T> Enum type
+     * @param in DataInput to read from
+     * @param enumType Class type of Enum
+     * @return Enum represented by String read from DataInput
+     * @throws IOException
+     */
+    public static <T extends Enum<T>> T readEnum(DataInput in, Class<T> enumType) throws IOException {
+        return T.valueOf(enumType, Text.readString(in));
+    }
+
+    /**
+     * writes String value of enum to DataOutput.
+     * @param out Dataoutput stream
+     * @param enumVal enum value
+     * @throws IOException
+     */
+    public static void writeEnum(DataOutput out, Enum<?> enumVal) throws IOException {
+        Text.writeString(out, enumVal.name());
+    }
+
+    /**
+     * Skip <i>len</i> number of bytes in input stream<i>in</i>
+     * @param in input stream
+     * @param len number of bytes to skip
+     * @throws IOException when skipped less number of bytes
+     */
+    public static void skipFully(DataInput in, int len) throws IOException {
+        int total = 0;
+        int cur = 0;
+
+        while ((total < len) && ((cur = in.skipBytes(len - total)) > 0)) {
+            total += cur;
+        }
+
+        if (total < len) {
+            throw new IOException("Not able to skip " + len + " bytes, possibly " + "due to end of input.");
+        }
+    }
+
+    /** Convert writables to a byte array */
+    public static byte[] toByteArray(Writable... writables) {
+        final DataOutputBuffer out = new DataOutputBuffer();
+        try {
+            for (Writable w : writables) {
+                w.write(out);
+            }
+            out.close();
+        } catch (IOException e) {
+            throw new RuntimeException("Fail to convert writables to a byte array", e);
+        }
+        return out.getData();
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/converters/DoubleWritableConverter.java

@@ -0,0 +1,36 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.converters;
+
+import org.datavec.api.io.WritableConverter;
+import org.datavec.api.writable.*;
+
+public class DoubleWritableConverter implements WritableConverter {
+    @Override
+    public Writable convert(Writable writable) throws WritableConverterException {
+        if (writable instanceof Text || writable instanceof FloatWritable || writable instanceof IntWritable
+                        || writable instanceof DoubleWritable) {
+            return new DoubleWritable(writable.toDouble());
+        }
+
+        throw new WritableConverterException("Unable to convert type " + writable.getClass());
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/converters/FloatWritableConverter.java

@@ -0,0 +1,36 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.converters;
+
+import org.datavec.api.io.WritableConverter;
+import org.datavec.api.writable.*;
+
+public class FloatWritableConverter implements WritableConverter {
+    @Override
+    public Writable convert(Writable writable) throws WritableConverterException {
+        if (writable instanceof Text || writable instanceof DoubleWritable || writable instanceof IntWritable
+                        || writable instanceof FloatWritable) {
+            return new FloatWritable(writable.toFloat());
+        }
+
+        throw new WritableConverterException("Unable to convert type " + writable.getClass());
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/converters/LabelWriterConverter.java

@@ -0,0 +1,41 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.converters;
+
+import org.datavec.api.io.WritableConverter;
+import org.datavec.api.writable.IntWritable;
+import org.datavec.api.writable.Writable;
+
+import java.util.List;
+
+public class LabelWriterConverter implements WritableConverter {
+    private List<String> labels;
+
+    public LabelWriterConverter(List<String> labels) {
+        this.labels = labels;
+    }
+
+    @Override
+    public Writable convert(Writable writable) throws WritableConverterException {
+        String label = writable.toString();
+        return new IntWritable(labels.indexOf(label));
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/converters/SelfWritableConverter.java

@@ -0,0 +1,31 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.converters;
+
+import org.datavec.api.io.WritableConverter;
+import org.datavec.api.writable.Writable;
+
+public class SelfWritableConverter implements WritableConverter {
+    @Override
+    public Writable convert(Writable writable) {
+        return writable;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/converters/WritableConverterException.java

@@ -0,0 +1,42 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.converters;
+
+public class WritableConverterException extends Exception {
+    public WritableConverterException() {}
+
+    public WritableConverterException(String message) {
+        super(message);
+    }
+
+    public WritableConverterException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public WritableConverterException(Throwable cause) {
+        super(cause);
+    }
+
+    public WritableConverterException(String message, Throwable cause, boolean enableSuppression,
+                    boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/filters/BalancedPathFilter.java

@@ -0,0 +1,143 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.filters;
+
+import org.datavec.api.io.labels.ParentPathLabelGenerator;
+import org.datavec.api.io.labels.PathLabelGenerator;
+import org.datavec.api.writable.Writable;
+
+import java.net.URI;
+import java.util.*;
+
+public class BalancedPathFilter extends RandomPathFilter {
+
+    protected PathLabelGenerator labelGenerator;
+    protected long maxLabels = 0, minPathsPerLabel = 0, maxPathsPerLabel = 0;
+    protected String[] labels = null;
+
+    /** Calls {@code this(random, extensions, labelGenerator, 0, 0, 0, 0)}. */
+    public BalancedPathFilter(Random random, String[] extensions, PathLabelGenerator labelGenerator) {
+        this(random, extensions, labelGenerator, 0, 0, 0, 0);
+    }
+
+    /** Calls {@code this(random, null, labelGenerator, 0, 0, 0, maxPathsPerLabel)}. */
+    public BalancedPathFilter(Random random, PathLabelGenerator labelGenerator, long maxPathsPerLabel) {
+        this(random, null, labelGenerator, 0, 0, 0, maxPathsPerLabel);
+    }
+
+    /** Calls {@code this(random, extensions, labelGenerator, 0, 0, 0, maxPathsPerLabel)}. */
+    public BalancedPathFilter(Random random, String[] extensions, PathLabelGenerator labelGenerator,
+                    long maxPathsPerLabel) {
+        this(random, extensions, labelGenerator, 0, 0, 0, maxPathsPerLabel);
+    }
+
+    /** Calls {@code this(random, extensions, labelGenerator, 0, maxLabels, 0, maxPathsPerLabel)}. */
+    public BalancedPathFilter(Random random, PathLabelGenerator labelGenerator, long maxPaths, long maxLabels,
+                    long maxPathsPerLabel) {
+        this(random, null, labelGenerator, maxPaths, maxLabels, 0, maxPathsPerLabel);
+    }
+
+    /** Calls {@code this(random, extensions, labelGenerator, 0, maxLabels, 0, maxPathsPerLabel)}. */
+    public BalancedPathFilter(Random random, String[] extensions, PathLabelGenerator labelGenerator, long maxLabels,
+                    long maxPathsPerLabel) {
+        this(random, extensions, labelGenerator, 0, maxLabels, 0, maxPathsPerLabel);
+    }
+
+    /**
+     * Constructs an instance of the PathFilter. If {@code minPathsPerLabel > 0},
+     * it might return an unbalanced set if the value is larger than the number of
+     * examples available for the label with the minimum amount.
+     *
+     * @param random           object to use
+     * @param extensions       of files to keep
+     * @param labelGenerator   to obtain labels from paths
+     * @param maxPaths         max number of paths to return (0 == unlimited)
+     * @param maxLabels        max number of labels to return (0 == unlimited)
+     * @param minPathsPerLabel min number of paths per labels to return
+     * @param maxPathsPerLabel max number of paths per labels to return (0 == unlimited)
+     * @param labels           of the paths to keep (empty set == keep all paths)
+     */
+    public BalancedPathFilter(Random random, String[] extensions, PathLabelGenerator labelGenerator, long maxPaths,
+                    long maxLabels, long minPathsPerLabel, long maxPathsPerLabel, String... labels) {
+        super(random, extensions, maxPaths);
+        this.labelGenerator = labelGenerator;
+        this.maxLabels = maxLabels;
+        this.minPathsPerLabel = minPathsPerLabel;
+        this.maxPathsPerLabel = maxPathsPerLabel;
+        this.labels = labels;
+    }
+
+    protected boolean acceptLabel(String name) {
+        if (labels == null || labels.length == 0) {
+            return true;
+        }
+        for (String label : labels) {
+            if (name.equals(label)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @Override
+    public URI[] filter(URI[] paths) {
+        paths = super.filter(paths);
+        if (labelGenerator == null)
+            labelGenerator = new ParentPathLabelGenerator();
+        Map<Writable, List<URI>> labelPaths = new LinkedHashMap<Writable, List<URI>>();
+        for (int i = 0; i < paths.length; i++) {
+            URI path = paths[i];
+            Writable label = labelGenerator.getLabelForPath(path);
+            if (!acceptLabel(label.toString())) {
+                continue;
+            }
+            List<URI> pathList = labelPaths.get(label);
+            if (pathList == null) {
+                if (maxLabels > 0 && labelPaths.size() >= maxLabels) {
+                    continue;
+                }
+                labelPaths.put(label, pathList = new ArrayList<URI>());
+            }
+            pathList.add(path);
+        }
+
+        int minCount = maxPathsPerLabel > 0 ?
+                (int)Math.min(maxPathsPerLabel, Integer.MAX_VALUE) : Integer.MAX_VALUE;
+        for (List<URI> pathList : labelPaths.values()) {
+            if (minCount > pathList.size()) {
+                minCount = pathList.size();
+            }
+        }
+        if (minCount < minPathsPerLabel) {
+            minCount = (int)Math.min(minPathsPerLabel, Integer.MAX_VALUE);
+        }
+
+        ArrayList<URI> newpaths = new ArrayList<URI>();
+        for (int i = 0; i < minCount; i++) {
+            for (List<URI> p : labelPaths.values()) {
+                if (i < p.size()) {
+                    newpaths.add(p.get(i));
+                }
+            }
+        }
+        return newpaths.toArray(new URI[newpaths.size()]);
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/filters/PathFilter.java

@@ -0,0 +1,34 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.filters;
+
+import java.net.URI;
+
+/**
+ * Filters an array of paths in some way.
+ *
+ * @author saudet
+ */
+public interface PathFilter {
+
+    URI[] filter(URI[] paths);
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/filters/RandomPathFilter.java

@@ -0,0 +1,87 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.filters;
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Random;
+
+/**
+ * Randomizes the order of paths in an array.
+ *
+ * @author saudet
+ */
+public class RandomPathFilter implements PathFilter {
+
+    protected Random random;
+    protected String[] extensions;
+    protected long maxPaths = 0;
+
+    /** Calls {@code this(random, extensions, 0)}. */
+    public RandomPathFilter(Random random, String... extensions) {
+        this(random, extensions, 0);
+    }
+
+    /**
+     * Constructs an instance of the PathFilter.
+     *
+     * @param random     object to use
+     * @param extensions of files to keep
+     * @param maxPaths   max number of paths to return (0 == unlimited)
+     */
+    public RandomPathFilter(Random random, String[] extensions, long maxPaths) {
+        this.random = random;
+        this.extensions = extensions;
+        this.maxPaths = maxPaths;
+    }
+
+    protected boolean accept(String name) {
+        if (extensions == null || extensions.length == 0) {
+            return true;
+        }
+        for (String extension : extensions) {
+            if (name.endsWith("." + extension)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @Override
+    public URI[] filter(URI[] paths) {
+        // shuffle before to avoid sampling bias
+        ArrayList<URI> paths2 = new ArrayList<URI>(Arrays.asList(paths));
+        Collections.shuffle(paths2, random);
+
+        ArrayList<URI> newpaths = new ArrayList<URI>();
+        for (URI path : paths2) {
+            if (accept(path.toString())) {
+                newpaths.add(path);
+            }
+            if (maxPaths > 0 && newpaths.size() >= maxPaths) {
+                break;
+            }
+        }
+        return newpaths.toArray(new URI[newpaths.size()]);
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/labels/ParentPathLabelGenerator.java

@@ -0,0 +1,50 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.labels;
+
+import org.apache.commons.io.FilenameUtils;
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+
+import java.io.File;
+import java.net.URI;
+
+public class ParentPathLabelGenerator implements PathLabelGenerator {
+
+    public ParentPathLabelGenerator() {}
+
+    @Override
+    public Writable getLabelForPath(String path) {
+        // Label is in the directory
+        String dirName = FilenameUtils.getName(new File(path).getParent());
+        return new Text(dirName);
+    }
+
+    @Override
+    public Writable getLabelForPath(URI uri) {
+        return getLabelForPath(new File(uri).toString());
+    }
+
+    @Override
+    public boolean inferLabelClasses() {
+        return true;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/labels/PathLabelGenerator.java

@@ -0,0 +1,45 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.labels;
+
+import org.datavec.api.writable.Writable;
+
+import java.io.Serializable;
+import java.net.URI;
+
+public interface PathLabelGenerator extends Serializable {
+
+    Writable getLabelForPath(String path);
+
+    Writable getLabelForPath(URI uri);
+
+    /**
+     * If true: infer the set of possible label classes, and convert these to integer indexes. If when true, the
+     * returned Writables should be text writables.<br>
+     * <br>
+     * For regression use cases (or PathLabelGenerator classification instances that do their own label -> integer
+     * assignment), this should return false.
+     *
+     * @return whether label classes should be inferred
+     */
+    boolean inferLabelClasses();
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/labels/PathMultiLabelGenerator.java

@@ -0,0 +1,36 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.labels;
+
+import org.datavec.api.writable.Writable;
+
+import java.io.Serializable;
+import java.util.List;
+
+public interface PathMultiLabelGenerator extends Serializable {
+
+    /**
+     * @param uriPath The file or URI path to get the label for
+     * @return A list of labels for the specified URI/path
+     */
+    List<Writable> getLabels(String uriPath);
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/labels/PatternPathLabelGenerator.java

@@ -0,0 +1,65 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.labels;
+
+import org.apache.commons.io.FilenameUtils;
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+
+import java.io.File;
+import java.net.URI;
+
+/**
+ * Returns a label derived from the base name of the path. Splits the base name
+ * of the path with the given regex pattern, and returns the patternPosition'th
+ * element of the array.
+ *
+ * @author saudet
+ */
+public class PatternPathLabelGenerator implements PathLabelGenerator {
+    protected String pattern; // Pattern to split and segment file name, pass in regex
+    protected int patternPosition = 0;
+
+    public PatternPathLabelGenerator(String pattern) {
+        this.pattern = pattern;
+    }
+
+    public PatternPathLabelGenerator(String pattern, int patternPosition) {
+        this.pattern = pattern;
+        this.patternPosition = patternPosition;
+    }
+
+    @Override
+    public Writable getLabelForPath(String path) {
+        // Label is in the filename
+        return new Text(FilenameUtils.getBaseName(path).split(pattern)[patternPosition]);
+    }
+
+    @Override
+    public Writable getLabelForPath(URI uri) {
+        return getLabelForPath(new File(uri).toString());
+    }
+
+    @Override
+    public boolean inferLabelClasses() {
+        return true;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/serializers/Deserializer.java

@@ -0,0 +1,48 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.serializers;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public interface Deserializer<T> {
+    /**
+     * <p>Prepare the deserializer for reading.</p>
+     */
+    void open(InputStream in) throws IOException;
+
+    /**
+     * <p>
+     * Deserialize the next object from the underlying input stream.
+     * If the object <code>t</code> is non-null then this deserializer
+     * <i>may</i> set its internal state to the next object read from the input
+     * stream. Otherwise, if the object <code>t</code> is null a new
+     * deserialized object will be created.
+     * </p>
+     * @return the deserialized object
+     */
+    T deserialize(T t) throws IOException;
+
+    /**
+     * <p>Close the underlying input stream and clear up any resources.</p>
+     */
+    void close() throws IOException;
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/serializers/Serialization.java

@@ -0,0 +1,42 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.serializers;
+
+public interface Serialization<T> {
+
+    /**
+     * Allows clients to test whether this {@link Serialization}
+     * supports the given class.
+     */
+    boolean accept(Class<?> c);
+
+    /**
+     * @return a {@link Serializer} for the given class.
+     */
+    Serializer<T> getSerializer(Class<T> c);
+
+    /**
+     * @return a {@link Deserializer} for the given class.
+     */
+    Deserializer<T> getDeserializer(Class<T> c);
+
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/serializers/SerializationFactory.java

@@ -0,0 +1,84 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.serializers;
+
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.conf.Configured;
+import org.datavec.api.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class SerializationFactory extends Configured {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SerializationFactory.class.getName());
+
+    private List<Serialization<?>> serializations = new ArrayList<>();
+
+    /**
+     * <p>
+     * Serializations are found by reading the <code>io.serializations</code>
+     * property from <code>conf</code>, which is a comma-delimited list of
+     * classnames.
+     * </p>
+     */
+    public SerializationFactory(Configuration conf) {
+        super(conf);
+        for (String serializerName : conf.getStrings("io.serializations",
+                        new String[] {"org.apache.hadoop.io.serializer.WritableSerialization"})) {
+            add(conf, serializerName);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private void add(Configuration conf, String serializationName) {
+        try {
+
+            Class<? extends Serialization> serializationClass =
+                            (Class<? extends Serialization>) conf.getClassByName(serializationName);
+            serializations.add(ReflectionUtils.newInstance(serializationClass, getConf()));
+        } catch (ClassNotFoundException e) {
+            LOG.warn("Serialization class not found: " + ExceptionUtils.getStackTrace(e));
+        }
+    }
+
+    public <T> Serializer<T> getSerializer(Class<T> c) {
+        return getSerialization(c).getSerializer(c);
+    }
+
+    public <T> Deserializer<T> getDeserializer(Class<T> c) {
+        return getSerialization(c).getDeserializer(c);
+    }
+
+    @SuppressWarnings("unchecked")
+    public <T> Serialization<T> getSerialization(Class<T> c) {
+        for (Serialization serialization : serializations) {
+            if (serialization.accept(c)) {
+                return (Serialization<T>) serialization;
+            }
+        }
+        return null;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/io/serializers/Serializer.java

@@ -0,0 +1,41 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.io.serializers;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public interface Serializer<T> {
+    /**
+     * <p>Prepare the serializer for writing.</p>
+     */
+    void open(OutputStream out) throws IOException;
+
+    /**
+     * <p>Serialize <code>t</code> to the underlying output stream.</p>
+     */
+    void serialize(T t) throws IOException;
+
+    /**
+     * <p>Close the underlying output stream and clear up any resources.</p>
+     */
+    void close() throws IOException;
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/Buffer.java

@@ -0,0 +1,241 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records;
+
+import java.io.UnsupportedEncodingException;
+
+public class Buffer implements Comparable, Cloneable {
+    /** Number of valid bytes in this.bytes. */
+    private int count;
+    /** Backing store for Buffer. */
+    private byte[] bytes = null;
+
+    /**
+     * Create a zero-count sequence.
+     */
+    public Buffer() {
+        this.count = 0;
+    }
+
+    /**
+     * Create a Buffer using the byte array as the initial value.
+     *
+     * @param bytes This array becomes the backing storage for the object.
+     */
+    public Buffer(byte[] bytes) {
+        this.bytes = bytes;
+        this.count = (bytes == null) ? 0 : bytes.length;
+    }
+
+    /**
+     * Create a Buffer using the byte range as the initial value.
+     *
+     * @param bytes Copy of this array becomes the backing storage for the object.
+     * @param offset offset into byte array
+     * @param length length of data
+     */
+    public Buffer(byte[] bytes, int offset, int length) {
+        copy(bytes, offset, length);
+    }
+
+
+    /**
+     * Use the specified bytes array as underlying sequence.
+     *
+     * @param bytes byte sequence
+     */
+    public void set(byte[] bytes) {
+        this.count = (bytes == null) ? 0 : bytes.length;
+        this.bytes = bytes;
+    }
+
+    /**
+     * Copy the specified byte array to the Buffer. Replaces the current buffer.
+     *
+     * @param bytes byte array to be assigned
+     * @param offset offset into byte array
+     * @param length length of data
+     */
+    public final void copy(byte[] bytes, int offset, int length) {
+        if (this.bytes == null || this.bytes.length < length) {
+            this.bytes = new byte[length];
+        }
+        System.arraycopy(bytes, offset, this.bytes, 0, length);
+        this.count = length;
+    }
+
+    /**
+     * Get the data from the Buffer.
+     *
+     * @return The data is only valid between 0 and getCount() - 1.
+     */
+    public byte[] get() {
+        if (bytes == null) {
+            bytes = new byte[0];
+        }
+        return bytes;
+    }
+
+    /**
+     * Get the current count of the buffer.
+     */
+    public int getCount() {
+        return count;
+    }
+
+    /**
+     * Get the capacity, which is the maximum count that could handled without
+     * resizing the backing storage.
+     *
+     * @return The number of bytes
+     */
+    public int getCapacity() {
+        return this.get().length;
+    }
+
+    /**
+     * Change the capacity of the backing storage.
+     * The data is preserved if newCapacity >= getCount().
+     * @param newCapacity The new capacity in bytes.
+     */
+    public void setCapacity(int newCapacity) {
+        if (newCapacity < 0) {
+            throw new IllegalArgumentException("Invalid capacity argument " + newCapacity);
+        }
+        if (newCapacity == 0) {
+            this.bytes = null;
+            this.count = 0;
+            return;
+        }
+        if (newCapacity != getCapacity()) {
+            byte[] data = new byte[newCapacity];
+            if (newCapacity < count) {
+                count = newCapacity;
+            }
+            if (count != 0) {
+                System.arraycopy(this.get(), 0, data, 0, count);
+            }
+            bytes = data;
+        }
+    }
+
+    /**
+     * Reset the buffer to 0 size
+     */
+    public void reset() {
+        setCapacity(0);
+    }
+
+    /**
+     * Change the capacity of the backing store to be the same as the current
+     * count of buffer.
+     */
+    public void truncate() {
+        setCapacity(count);
+    }
+
+    /**
+     * Append specified bytes to the buffer.
+     *
+     * @param bytes byte array to be appended
+     * @param offset offset into byte array
+     * @param length length of data
+    
+     */
+    public void append(byte[] bytes, int offset, int length) {
+        setCapacity(count + length);
+        System.arraycopy(bytes, offset, this.get(), count, length);
+        count = count + length;
+    }
+
+    /**
+     * Append specified bytes to the buffer
+     *
+     * @param bytes byte array to be appended
+     */
+    public void append(byte[] bytes) {
+        append(bytes, 0, bytes.length);
+    }
+
+    // inherit javadoc
+    public int hashCode() {
+        int hash = 1;
+        byte[] b = this.get();
+        for (int i = 0; i < count; i++)
+            hash = (31 * hash) + (int) b[i];
+        return hash;
+    }
+
+    /**
+     * Define the sort order of the Buffer.
+     *
+     * @param other The other buffer
+     * @return Positive if this is bigger than other, 0 if they are equal, and
+     *         negative if this is smaller than other.
+     */
+    public int compareTo(Object other) {
+        Buffer right = ((Buffer) other);
+        byte[] lb = this.get();
+        byte[] rb = right.get();
+        for (int i = 0; i < count && i < right.count; i++) {
+            int a = (lb[i] & 0xff);
+            int b = (rb[i] & 0xff);
+            if (a != b) {
+                return a - b;
+            }
+        }
+        return count - right.count;
+    }
+
+    // inherit javadoc
+    public boolean equals(Object other) {
+        if (other instanceof Buffer && this != other) {
+            return compareTo(other) == 0;
+        }
+        return (this == other);
+    }
+
+    // inheric javadoc
+    public String toString() {
+        StringBuilder sb = new StringBuilder(2 * count);
+        for (int idx = 0; idx < count; idx++) {
+            sb.append(Character.forDigit((bytes[idx] & 0xF0) >> 4, 16));
+            sb.append(Character.forDigit(bytes[idx] & 0x0F, 16));
+        }
+        return sb.toString();
+    }
+
+    /**
+     * Convert the byte buffer to a string an specific character encoding
+     *
+     * @param charsetName Valid Java Character Set Name
+     */
+    public String toString(String charsetName) throws UnsupportedEncodingException {
+        return new String(this.get(), 0, this.getCount(), charsetName);
+    }
+
+    // inherit javadoc
+    public Object clone() throws CloneNotSupportedException {
+        Buffer result = (Buffer) super.clone();
+        result.copy(this.get(), 0, this.getCount());
+        return result;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/IOUtils.java

@@ -0,0 +1,476 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records;
+
+import org.datavec.api.io.WritableComparator;
+import org.datavec.api.io.WritableUtils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class IOUtils {
+
+    /** Cannot create a new instance of IOUtils */
+    private IOUtils() {}
+
+    public static final char[] hexchars =
+                    {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'};
+
+    /**
+     *
+     * @param s
+     * @return
+     */
+    static String toXMLString(String s) {
+        StringBuilder sb = new StringBuilder();
+        for (int idx = 0; idx < s.length(); idx++) {
+            char ch = s.charAt(idx);
+            if (ch == '<') {
+                sb.append("&lt;");
+            } else if (ch == '&') {
+                sb.append("&amp;");
+            } else if (ch == '%') {
+                sb.append("%0025");
+            } else if (ch < 0x20 || (ch > 0xD7FF && ch < 0xE000) || (ch > 0xFFFD)) {
+                sb.append("%");
+                sb.append(hexchars[(ch & 0xF000) >> 12]);
+                sb.append(hexchars[(ch & 0x0F00) >> 8]);
+                sb.append(hexchars[(ch & 0x00F0) >> 4]);
+                sb.append(hexchars[(ch & 0x000F)]);
+            } else {
+                sb.append(ch);
+            }
+        }
+        return sb.toString();
+    }
+
+    static private int h2c(char ch) {
+        if (ch >= '0' && ch <= '9') {
+            return ch - '0';
+        } else if (ch >= 'A' && ch <= 'F') {
+            return ch - 'A' + 10;
+        } else if (ch >= 'a' && ch <= 'f') {
+            return ch - 'a' + 10;
+        }
+        return 0;
+    }
+
+    /**
+     *
+     * @param s
+     * @return
+     */
+    static String fromXMLString(String s) {
+        StringBuilder sb = new StringBuilder();
+        for (int idx = 0; idx < s.length();) {
+            char ch = s.charAt(idx++);
+            if (ch == '%') {
+                int ch1 = h2c(s.charAt(idx++)) << 12;
+                int ch2 = h2c(s.charAt(idx++)) << 8;
+                int ch3 = h2c(s.charAt(idx++)) << 4;
+                int ch4 = h2c(s.charAt(idx++));
+                char res = (char) (ch1 | ch2 | ch3 | ch4);
+                sb.append(res);
+            } else {
+                sb.append(ch);
+            }
+        }
+        return sb.toString();
+    }
+
+    /**
+     *
+     * @param s
+     * @return
+     */
+    static String toCSVString(String s) {
+        StringBuilder sb = new StringBuilder(s.length() + 1);
+        sb.append('\'');
+        int len = s.length();
+        for (int i = 0; i < len; i++) {
+            char c = s.charAt(i);
+            switch (c) {
+                case '\0':
+                    sb.append("%00");
+                    break;
+                case '\n':
+                    sb.append("%0A");
+                    break;
+                case '\r':
+                    sb.append("%0D");
+                    break;
+                case ',':
+                    sb.append("%2C");
+                    break;
+                case '}':
+                    sb.append("%7D");
+                    break;
+                case '%':
+                    sb.append("%25");
+                    break;
+                default:
+                    sb.append(c);
+            }
+        }
+        return sb.toString();
+    }
+
+    /**
+     *
+     * @param s
+     * @throws java.io.IOException
+     * @return
+     */
+    static String fromCSVString(String s) throws IOException {
+        if (s.charAt(0) != '\'') {
+            throw new IOException("Error deserializing string.");
+        }
+        int len = s.length();
+        StringBuilder sb = new StringBuilder(len - 1);
+        for (int i = 1; i < len; i++) {
+            char c = s.charAt(i);
+            if (c == '%') {
+                char ch1 = s.charAt(i + 1);
+                char ch2 = s.charAt(i + 2);
+                i += 2;
+                if (ch1 == '0' && ch2 == '0') {
+                    sb.append('\0');
+                } else if (ch1 == '0' && ch2 == 'A') {
+                    sb.append('\n');
+                } else if (ch1 == '0' && ch2 == 'D') {
+                    sb.append('\r');
+                } else if (ch1 == '2' && ch2 == 'C') {
+                    sb.append(',');
+                } else if (ch1 == '7' && ch2 == 'D') {
+                    sb.append('}');
+                } else if (ch1 == '2' && ch2 == '5') {
+                    sb.append('%');
+                } else {
+                    throw new IOException("Error deserializing string.");
+                }
+            } else {
+                sb.append(c);
+            }
+        }
+        return sb.toString();
+    }
+
+    /**
+     *
+     * @param s
+     * @return
+     */
+    static String toXMLBuffer(Buffer s) {
+        return s.toString();
+    }
+
+    /**
+     *
+     * @param s
+     * @throws java.io.IOException
+     * @return
+     */
+    static Buffer fromXMLBuffer(String s) throws IOException {
+        if (s.length() == 0) {
+            return new Buffer();
+        }
+        int blen = s.length() / 2;
+        byte[] barr = new byte[blen];
+        for (int idx = 0; idx < blen; idx++) {
+            char c1 = s.charAt(2 * idx);
+            char c2 = s.charAt(2 * idx + 1);
+            barr[idx] = (byte) Integer.parseInt("" + c1 + c2, 16);
+        }
+        return new Buffer(barr);
+    }
+
+    /**
+     *
+     * @param buf
+     * @return
+     */
+    static String toCSVBuffer(Buffer buf) {
+        StringBuilder sb = new StringBuilder("#");
+        sb.append(buf.toString());
+        return sb.toString();
+    }
+
+    /**
+     * Converts a CSV-serialized representation of buffer to a new
+     * Buffer
+     * @param s CSV-serialized representation of buffer
+     * @throws java.io.IOException
+     * @return Deserialized Buffer
+     */
+    static Buffer fromCSVBuffer(String s) throws IOException {
+        if (s.charAt(0) != '#') {
+            throw new IOException("Error deserializing buffer.");
+        }
+        if (s.length() == 1) {
+            return new Buffer();
+        }
+        int blen = (s.length() - 1) / 2;
+        byte[] barr = new byte[blen];
+        for (int idx = 0; idx < blen; idx++) {
+            char c1 = s.charAt(2 * idx + 1);
+            char c2 = s.charAt(2 * idx + 2);
+            barr[idx] = (byte) Integer.parseInt("" + c1 + c2, 16);
+        }
+        return new Buffer(barr);
+    }
+
+    private static int utf8LenForCodePoint(final int cpt) throws IOException {
+        if (cpt >= 0 && cpt <= 0x7F) {
+            return 1;
+        }
+        if (cpt >= 0x80 && cpt <= 0x07FF) {
+            return 2;
+        }
+        if ((cpt >= 0x0800 && cpt < 0xD800) || (cpt > 0xDFFF && cpt <= 0xFFFD)) {
+            return 3;
+        }
+        if (cpt >= 0x10000 && cpt <= 0x10FFFF) {
+            return 4;
+        }
+        throw new IOException("Illegal Unicode Codepoint " + Integer.toHexString(cpt) + " in string.");
+    }
+
+    private static final int B10 = Integer.parseInt("10000000", 2);
+    private static final int B110 = Integer.parseInt("11000000", 2);
+    private static final int B1110 = Integer.parseInt("11100000", 2);
+    private static final int B11110 = Integer.parseInt("11110000", 2);
+    private static final int B11 = Integer.parseInt("11000000", 2);
+    private static final int B111 = Integer.parseInt("11100000", 2);
+    private static final int B1111 = Integer.parseInt("11110000", 2);
+    private static final int B11111 = Integer.parseInt("11111000", 2);
+
+    private static int writeUtf8(int cpt, final byte[] bytes, final int offset) throws IOException {
+        if (cpt >= 0 && cpt <= 0x7F) {
+            bytes[offset] = (byte) cpt;
+            return 1;
+        }
+        if (cpt >= 0x80 && cpt <= 0x07FF) {
+            bytes[offset + 1] = (byte) (B10 | (cpt & 0x3F));
+            cpt = cpt >> 6;
+            bytes[offset] = (byte) (B110 | (cpt & 0x1F));
+            return 2;
+        }
+        if ((cpt >= 0x0800 && cpt < 0xD800) || (cpt > 0xDFFF && cpt <= 0xFFFD)) {
+            bytes[offset + 2] = (byte) (B10 | (cpt & 0x3F));
+            cpt = cpt >> 6;
+            bytes[offset + 1] = (byte) (B10 | (cpt & 0x3F));
+            cpt = cpt >> 6;
+            bytes[offset] = (byte) (B1110 | (cpt & 0x0F));
+            return 3;
+        }
+        if (cpt >= 0x10000 && cpt <= 0x10FFFF) {
+            bytes[offset + 3] = (byte) (B10 | (cpt & 0x3F));
+            cpt = cpt >> 6;
+            bytes[offset + 2] = (byte) (B10 | (cpt & 0x3F));
+            cpt = cpt >> 6;
+            bytes[offset + 1] = (byte) (B10 | (cpt & 0x3F));
+            cpt = cpt >> 6;
+            bytes[offset] = (byte) (B11110 | (cpt & 0x07));
+            return 4;
+        }
+        throw new IOException("Illegal Unicode Codepoint " + Integer.toHexString(cpt) + " in string.");
+    }
+
+    static void toBinaryString(final DataOutput out, final String str) throws IOException {
+        final int strlen = str.length();
+        byte[] bytes = new byte[strlen * 4]; // Codepoints expand to 4 bytes max
+        int utf8Len = 0;
+        int idx = 0;
+        while (idx < strlen) {
+            final int cpt = str.codePointAt(idx);
+            idx += Character.isSupplementaryCodePoint(cpt) ? 2 : 1;
+            utf8Len += writeUtf8(cpt, bytes, utf8Len);
+        }
+        writeVInt(out, utf8Len);
+        out.write(bytes, 0, utf8Len);
+    }
+
+    static boolean isValidCodePoint(int cpt) {
+        return !((cpt > 0x10FFFF) || (cpt >= 0xD800 && cpt <= 0xDFFF) || (cpt >= 0xFFFE && cpt <= 0xFFFF));
+    }
+
+    private static int utf8ToCodePoint(int b1, int b2, int b3, int b4) {
+        int cpt;
+        cpt = (((b1 & ~B11111) << 18) | ((b2 & ~B11) << 12) | ((b3 & ~B11) << 6) | (b4 & ~B11));
+        return cpt;
+    }
+
+    private static int utf8ToCodePoint(int b1, int b2, int b3) {
+        int cpt = 0;
+        cpt = (((b1 & ~B1111) << 12) | ((b2 & ~B11) << 6) | (b3 & ~B11));
+        return cpt;
+    }
+
+    private static int utf8ToCodePoint(int b1, int b2) {
+        int cpt = 0;
+        cpt = (((b1 & ~B111) << 6) | (b2 & ~B11));
+        return cpt;
+    }
+
+    private static void checkB10(int b) throws IOException {
+        if ((b & B11) != B10) {
+            throw new IOException("Invalid UTF-8 representation.");
+        }
+    }
+
+    static String fromBinaryString(final DataInput din) throws IOException {
+        final int utf8Len = readVInt(din);
+        final byte[] bytes = new byte[utf8Len];
+        din.readFully(bytes);
+        int len = 0;
+        // For the most commmon case, i.e. ascii, numChars = utf8Len
+        StringBuilder sb = new StringBuilder(utf8Len);
+        while (len < utf8Len) {
+            int cpt;
+            final int b1 = bytes[len++] & 0xFF;
+            if (b1 <= 0x7F) {
+                cpt = b1;
+            } else if ((b1 & B11111) == B11110) {
+                int b2 = bytes[len++] & 0xFF;
+                checkB10(b2);
+                int b3 = bytes[len++] & 0xFF;
+                checkB10(b3);
+                int b4 = bytes[len++] & 0xFF;
+                checkB10(b4);
+                cpt = utf8ToCodePoint(b1, b2, b3, b4);
+            } else if ((b1 & B1111) == B1110) {
+                int b2 = bytes[len++] & 0xFF;
+                checkB10(b2);
+                int b3 = bytes[len++] & 0xFF;
+                checkB10(b3);
+                cpt = utf8ToCodePoint(b1, b2, b3);
+            } else if ((b1 & B111) == B110) {
+                int b2 = bytes[len++] & 0xFF;
+                checkB10(b2);
+                cpt = utf8ToCodePoint(b1, b2);
+            } else {
+                throw new IOException("Invalid UTF-8 byte " + Integer.toHexString(b1) + " at offset " + (len - 1)
+                                + " in length of " + utf8Len);
+            }
+            if (!isValidCodePoint(cpt)) {
+                throw new IOException("Illegal Unicode Codepoint " + Integer.toHexString(cpt) + " in stream.");
+            }
+            sb.appendCodePoint(cpt);
+        }
+        return sb.toString();
+    }
+
+    /** Parse a float from a byte array. */
+    public static float readFloat(byte[] bytes, int start) {
+        return WritableComparator.readFloat(bytes, start);
+    }
+
+    /** Parse a double from a byte array. */
+    public static double readDouble(byte[] bytes, int start) {
+        return WritableComparator.readDouble(bytes, start);
+    }
+
+    /**
+     * Reads a zero-compressed encoded long from a byte array and returns it.
+     * @param bytes byte array with decode long
+     * @param start starting index
+     * @throws java.io.IOException
+     * @return deserialized long
+     */
+    public static long readVLong(byte[] bytes, int start) throws IOException {
+        return WritableComparator.readVLong(bytes, start);
+    }
+
+    /**
+     * Reads a zero-compressed encoded integer from a byte array and returns it.
+     * @param bytes byte array with the encoded integer
+     * @param start start index
+     * @throws java.io.IOException
+     * @return deserialized integer
+     */
+    public static int readVInt(byte[] bytes, int start) throws IOException {
+        return WritableComparator.readVInt(bytes, start);
+    }
+
+    /**
+     * Reads a zero-compressed encoded long from a stream and return it.
+     * @param in input stream
+     * @throws java.io.IOException
+     * @return deserialized long
+     */
+    public static long readVLong(DataInput in) throws IOException {
+        return WritableUtils.readVLong(in);
+    }
+
+    /**
+     * Reads a zero-compressed encoded integer from a stream and returns it.
+     * @param in input stream
+     * @throws java.io.IOException
+     * @return deserialized integer
+     */
+    public static int readVInt(DataInput in) throws IOException {
+        return WritableUtils.readVInt(in);
+    }
+
+    /**
+     * Get the encoded length if an integer is stored in a variable-length format
+     * @return the encoded length
+     */
+    public static int getVIntSize(long i) {
+        return WritableUtils.getVIntSize(i);
+    }
+
+    /**
+     * Serializes a long to a binary stream with zero-compressed encoding.
+     * For -112 <= i <= 127, only one byte is used with the actual value.
+     * For other values of i, the first byte value indicates whether the
+     * long is positive or negative, and the number of bytes that follow.
+     * If the first byte value v is between -113 and -120, the following long
+     * is positive, with number of bytes that follow are -(v+112).
+     * If the first byte value v is between -121 and -128, the following long
+     * is negative, with number of bytes that follow are -(v+120). Bytes are
+     * stored in the high-non-zero-byte-first order.
+     *
+     * @param stream Binary output stream
+     * @param i Long to be serialized
+     * @throws java.io.IOException
+     */
+    public static void writeVLong(DataOutput stream, long i) throws IOException {
+        WritableUtils.writeVLong(stream, i);
+    }
+
+    /**
+     * Serializes an int to a binary stream with zero-compressed encoding.
+     *
+     * @param stream Binary output stream
+     * @param i int to be serialized
+     * @throws java.io.IOException
+     */
+    public static void writeVInt(DataOutput stream, int i) throws IOException {
+        WritableUtils.writeVInt(stream, i);
+    }
+
+    /** Lexicographic order of binary data. */
+    public static int compareBytes(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+        return WritableComparator.compareBytes(b1, s1, l1, b2, s2, l2);
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/Index.java

@@ -0,0 +1,27 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records;
+
+public interface Index {
+    boolean done();
+
+    void incr();
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/Record.java

@@ -0,0 +1,55 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records;
+
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.writable.Writable;
+
+import java.io.Serializable;
+import java.util.List;
+
+public interface Record extends Serializable {
+
+    /**
+     * Get the record values, as a {@code List<Writable>}
+     *
+     * @return Record values
+     */
+    List<Writable> getRecord();
+
+    /**
+     * Get the record values for this Record
+     */
+    void setRecord(List<Writable> record);
+
+    /**
+     * Get the RecordMetaData for this record
+     *
+     * @return Metadata for this record (or null, if none has been set)
+     */
+    RecordMetaData getMetaData();
+
+    /**
+     * Set the Record metadata
+     */
+    void setMetaData(RecordMetaData recordMetaData);
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/SequenceRecord.java

@@ -0,0 +1,73 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records;
+
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.writable.Writable;
+
+import java.io.Serializable;
+import java.util.List;
+
+public interface SequenceRecord extends Serializable {
+
+    /**
+     * Get the sequence record values
+     *
+     * @return Sequence record values
+     */
+    List<List<Writable>> getSequenceRecord();
+
+    /**
+     * Get the overall length of the sequence record (number of time/sequence steps, etc).
+     * Equivalent to {@code getSequenceRecord().size()}
+     *
+     * @return Length of sequence record
+     */
+    int getSequenceLength();
+
+    /**
+     * Get a single time step. Equivalent to {@code getSequenceRecord().get(timeStep)}
+     *
+     * @param timeStep Time step to get. Must be {@code 0 <= timeStep < getSequenceLength()}
+     * @return Values for a single time step
+     */
+    List<Writable> getTimeStep(int timeStep);
+
+    /**
+     * Set the sequence record values
+     *
+     * @param sequenceRecord Sequence record values to set
+     */
+    void setSequenceRecord(List<List<Writable>> sequenceRecord);
+
+    /**
+     * Get the RecordMetaData for this record
+     *
+     * @return Metadata for this record (or null, if none has been set)
+     */
+    RecordMetaData getMetaData();
+
+    /**
+     * Set the Record metadata
+     */
+    void setMetaData(RecordMetaData recordMetaData);
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/converter/RecordReaderConverter.java

@@ -0,0 +1,106 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.converter;
+
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.records.reader.SequenceRecordReader;
+import org.datavec.api.records.writer.RecordWriter;
+import org.datavec.api.records.writer.SequenceRecordWriter;
+
+import java.io.IOException;
+
+public class RecordReaderConverter {
+
+    private RecordReaderConverter() { }
+
+    /**
+     * Write all values from the specified record reader to the specified record writer.
+     * Closes the record writer on completion
+     *
+     * @param reader Record reader (source of data)
+     * @param writer Record writer (location to write data)
+     * @throws IOException If underlying reader/writer throws an exception
+     */
+    public static void convert(RecordReader reader, RecordWriter writer) throws IOException {
+        convert(reader, writer, true);
+    }
+
+    /**
+     * Write all values from the specified record reader to the specified record writer.
+     * Optionally, close the record writer on completion
+     *
+     * @param reader Record reader (source of data)
+     * @param writer Record writer (location to write data)
+     * @param closeOnCompletion if true: close the record writer once complete, via {@link RecordWriter#close()}
+     * @throws IOException If underlying reader/writer throws an exception
+     */
+    public static void convert(RecordReader reader, RecordWriter writer, boolean closeOnCompletion) throws IOException {
+
+        if(!reader.hasNext()){
+            throw new UnsupportedOperationException("Cannot convert RecordReader: reader has no next element");
+        }
+
+        while(reader.hasNext()){
+            writer.write(reader.next());
+        }
+
+        if(closeOnCompletion){
+            writer.close();
+        }
+    }
+
+    /**
+     * Write all sequences from the specified sequence record reader to the specified sequence record writer.
+     * Closes the sequence record writer on completion.
+     *
+     * @param reader Sequence record reader (source of data)
+     * @param writer Sequence record writer (location to write data)
+     * @throws IOException If underlying reader/writer throws an exception
+     */
+    public static void convert(SequenceRecordReader reader, SequenceRecordWriter writer) throws IOException {
+        convert(reader, writer, true);
+    }
+
+    /**
+     * Write all sequences from the specified sequence record reader to the specified sequence record writer.
+     * Closes the sequence record writer on completion.
+     *
+     * @param reader Sequence record reader (source of data)
+     * @param writer Sequence record writer (location to write data)
+     * @param closeOnCompletion if true: close the record writer once complete, via {@link SequenceRecordWriter#close()}
+     * @throws IOException If underlying reader/writer throws an exception
+     */
+    public static void convert(SequenceRecordReader reader, SequenceRecordWriter writer, boolean closeOnCompletion) throws IOException {
+
+        if(!reader.hasNext()){
+            throw new UnsupportedOperationException("Cannot convert SequenceRecordReader: reader has no next element");
+        }
+
+        while(reader.hasNext()){
+            writer.write(reader.sequenceRecord());
+        }
+
+        if(closeOnCompletion){
+            writer.close();
+        }
+    }
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/impl/Record.java

@@ -0,0 +1,37 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.impl;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.writable.Writable;
+
+import java.util.List;
+
+@AllArgsConstructor
+@Data
+public class Record implements org.datavec.api.records.Record {
+
+    private List<Writable> record;
+    private RecordMetaData metaData;
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/impl/SequenceRecord.java

@@ -0,0 +1,52 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.impl;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.writable.Writable;
+
+import java.util.List;
+
+@AllArgsConstructor
+@Data
+public class SequenceRecord implements org.datavec.api.records.SequenceRecord {
+
+    private List<List<Writable>> sequenceRecord;
+    private RecordMetaData metaData;
+
+    @Override
+    public int getSequenceLength() {
+        if (sequenceRecord == null)
+            return 0;
+        return sequenceRecord.size();
+    }
+
+    @Override
+    public List<Writable> getTimeStep(int timeStep) {
+        if (timeStep < 0 || timeStep > sequenceRecord.size()) {
+            throw new IllegalArgumentException("Invalid input: " + sequenceRecord.size()
+                            + " time steps available; cannot get " + timeStep);
+        }
+        return sequenceRecord.get(timeStep);
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/listener/RecordListener.java

@@ -0,0 +1,52 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.listener;
+
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.records.writer.RecordWriter;
+
+import java.io.Serializable;
+
+public interface RecordListener extends Serializable {
+    /**
+     * Get if listener invoked.
+     */
+    boolean invoked();
+
+    /**
+     * Change invoke to true.
+     */
+    void invoke();
+
+    /**
+     * Event listener for each record to be read.
+     * @param reader the record reader
+     * @param record in raw format (Collection, File, String, Writable, etc)
+     */
+    void recordRead(RecordReader reader, Object record);
+
+    /**
+     * Event listener for each record to be written.
+     * @param writer the record writer
+     * @param record in raw format (Collection, File, String, Writable, etc)
+     */
+    void recordWrite(RecordWriter writer, Object record);
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/listener/impl/LogRecordListener.java

@@ -0,0 +1,54 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.listener.impl;
+
+import org.datavec.api.records.listener.RecordListener;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.records.writer.RecordWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LogRecordListener implements RecordListener {
+    private static final Logger log = LoggerFactory.getLogger(LogRecordListener.class);
+    private boolean invoked = false;
+
+    @Override
+    public boolean invoked() {
+        return invoked;
+    }
+
+    @Override
+    public void invoke() {
+        this.invoked = true;
+    }
+
+    @Override
+    public void recordRead(RecordReader reader, Object record) {
+        invoke();
+        log.info("Reading " + record);
+    }
+
+    @Override
+    public void recordWrite(RecordWriter writer, Object record) {
+        invoke();
+        log.info("Writing " + record);
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/mapper/RecordMapper.java

@@ -0,0 +1,160 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.mapper;
+
+import lombok.Builder;
+import lombok.Getter;
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.records.writer.RecordWriter;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.split.partition.NumberOfRecordsPartitioner;
+import org.datavec.api.split.partition.Partitioner;
+import org.datavec.api.writable.Writable;
+
+import java.util.List;
+
+@Builder
+public class RecordMapper {
+
+    private RecordReader recordReader;
+    private RecordWriter recordWriter;
+    private InputSplit inputUrl;
+    private InputSplit[] splitPerReader;
+    private RecordReader[] readersToConcat;
+
+    private InputSplit outputUrl;
+    @Builder.Default
+    private boolean callInitRecordReader = true;
+    @Builder.Default
+    private boolean callInitRecordWriter = true;
+    @Builder.Default
+    private boolean callInitPartitioner = true;
+    @Builder.Default
+    private Configuration configuration = new Configuration();
+
+    private Configuration[] configurationsPerReader;
+
+    @Getter
+    @Builder.Default
+    private Partitioner partitioner = new NumberOfRecordsPartitioner();
+    private int batchSize;
+
+    /**
+     * Copy the {@link RecordReader}
+     * data using the {@link RecordWriter}.
+     * Note that unless batch is supported by
+     * both the {@link RecordReader} and {@link RecordWriter}
+     * then writes will happen one at a time.
+     * You can see if batch is enabled via {@link RecordReader#batchesSupported()}
+     * and {@link RecordWriter#supportsBatch()} respectively.
+     * @throws Exception
+     */
+    public void copy() throws Exception {
+        if(callInitRecordReader) {
+            if(recordReader != null) {
+                recordReader.initialize(configuration, inputUrl);
+            }
+            else {
+                if(readersToConcat == null || splitPerReader == null)  {
+                    throw new IllegalArgumentException("No readers or input  splits found.");
+                }
+
+                if(readersToConcat.length != splitPerReader.length) {
+                    throw new IllegalArgumentException("One input split must be specified per record reader");
+                }
+
+                for(int i = 0; i < readersToConcat.length; i++) {
+                    if(readersToConcat[i] == null) {
+                        throw new IllegalStateException("Reader at record " + i + " was null!");
+                    }
+                    if(splitPerReader[i] == null) {
+                        throw new IllegalStateException("Split at " + i + " is null!");
+                    }
+                    //allow for, but do not enforce configurations per reader.
+                    if(configurationsPerReader != null) {
+                        readersToConcat[i].initialize(configurationsPerReader[i], splitPerReader[i]);
+                    }
+                    else {
+                        readersToConcat[i].initialize(configuration,splitPerReader[i]);
+                    }
+                }
+            }
+        }
+
+        if(callInitPartitioner) {
+            partitioner.init(configuration, outputUrl);
+        }
+
+        if(callInitRecordWriter) {
+            recordWriter.initialize(configuration, outputUrl, partitioner);
+        }
+
+        if(recordReader != null) {
+            write(recordReader,true);
+        }
+        else if(readersToConcat != null) {
+            for(RecordReader recordReader : readersToConcat) {
+                write(recordReader,false);
+            }
+
+            //close since we can't do it within the method
+            recordWriter.close();
+        }
+
+    }
+
+
+    private void write(RecordReader recordReader,boolean closeWriter) throws Exception {
+        if(batchSize > 0 && recordReader.batchesSupported() && recordWriter.supportsBatch()) {
+            while (recordReader.hasNext()) {
+                List<List<Writable>> next = recordReader.next(batchSize);
+                //ensure we can write a file for either the current or next iterations
+                if (partitioner.needsNewPartition()) {
+                    partitioner.currentOutputStream().flush();
+                    partitioner.currentOutputStream().close();
+                    partitioner.openNewStream();
+                }
+                //update records written
+                partitioner.updatePartitionInfo(recordWriter.writeBatch(next));
+
+            }
+
+            partitioner.currentOutputStream().flush();
+            recordReader.close();
+            if (closeWriter) {
+                partitioner.currentOutputStream().close();
+                recordWriter.close();
+            }
+        }
+
+        else {
+            while(recordReader.hasNext()) {
+                List<Writable> next = recordReader.next();
+                //update records written
+                partitioner.updatePartitionInfo(recordWriter.write(next));
+                if(partitioner.needsNewPartition()) {
+                    partitioner.openNewStream();
+                }
+            }
+        }
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaData.java

@@ -0,0 +1,45 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.metadata;
+
+import java.io.Serializable;
+import java.net.URI;
+
+public interface RecordMetaData extends Serializable {
+
+    /**
+     * Get a human-readable location for the data
+     */
+    String getLocation();
+
+    /**
+     * Return the URI for the source of the record
+     *
+     * @return The URI for the record (file, etc) - or null otherwise
+     */
+    URI getURI();
+
+    /**
+     * Get the class that was used to generate the record
+     */
+    Class<?> getReaderClass();
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataComposable.java

@@ -0,0 +1,66 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.metadata;
+
+import lombok.Data;
+
+import java.net.URI;
+
+@Data
+public class RecordMetaDataComposable implements RecordMetaData {
+
+    private Class<?> readerClass;
+    private RecordMetaData[] meta;
+
+    public RecordMetaDataComposable(RecordMetaData... recordMetaDatas) {
+        this(null, recordMetaDatas);
+    }
+
+    public RecordMetaDataComposable(Class<?> readerClass, RecordMetaData... recordMetaDatas) {
+        this.readerClass = readerClass;
+        this.meta = recordMetaDatas;
+    }
+
+    @Override
+    public String getLocation() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("locations(");
+        boolean first = true;
+        for (RecordMetaData rmd : meta) {
+            if (!first)
+                sb.append(",");
+            sb.append(rmd.getLocation());
+            first = false;
+        }
+        sb.append(")");
+        return sb.toString();
+    }
+
+    @Override
+    public URI getURI() {
+        return meta[0].getURI();
+    }
+
+    @Override
+    public Class<?> getReaderClass() {
+        return readerClass;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataComposableMap.java

@@ -0,0 +1,69 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.metadata;
+
+import lombok.Data;
+
+import java.net.URI;
+import java.util.Map;
+
+@Data
+public class RecordMetaDataComposableMap implements RecordMetaData {
+
+    private Class<?> readerClass;
+    private Map<String, RecordMetaData> meta;
+
+    public RecordMetaDataComposableMap(Map<String, RecordMetaData> recordMetaDatas) {
+        this(null, recordMetaDatas);
+    }
+
+    public RecordMetaDataComposableMap(Class<?> readerClass, Map<String, RecordMetaData> recordMetaDatas) {
+        this.readerClass = readerClass;
+        this.meta = recordMetaDatas;
+    }
+
+    @Override
+    public String getLocation() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("locations(");
+        boolean first = true;
+        for (Map.Entry<String, RecordMetaData> rmd : meta.entrySet()) {
+            if (!first)
+                sb.append(",");
+            sb.append(rmd.getKey()).append("=");
+            sb.append(rmd.getValue().getLocation());
+            first = false;
+        }
+        sb.append(")");
+        return sb.toString();
+    }
+
+    @Override
+    public URI getURI() {
+        String first = meta.keySet().iterator().next();
+        return meta.get(first).getURI();
+    }
+
+    @Override
+    public Class<?> getReaderClass() {
+        return readerClass;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataImageURI.java

@@ -0,0 +1,39 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.metadata;
+
+import java.net.URI;
+import lombok.Data;
+
+@Data
+public class RecordMetaDataImageURI extends RecordMetaDataURI {
+
+    private int origC;
+    private int origH;
+    private int origW;
+
+    public RecordMetaDataImageURI(URI uri, Class<?> readerClass, int origC, int origH, int origW) {
+        super(uri, readerClass);
+        this.origC = origC;
+        this.origH = origH;
+        this.origW = origW;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataIndex.java

@@ -0,0 +1,50 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.metadata;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.net.URI;
+
+@AllArgsConstructor
+@Data
+public class RecordMetaDataIndex implements RecordMetaData {
+
+    private final long index;
+    private final URI uri;
+    private final Class<?> readerClass;
+
+    @Override
+    public String getLocation() {
+        return "index=" + index;
+    }
+
+    @Override
+    public URI getURI() {
+        return uri;
+    }
+
+    @Override
+    public Class<?> getReaderClass() {
+        return readerClass;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataInterval.java

@@ -0,0 +1,55 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.metadata;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.net.URI;
+
+@AllArgsConstructor
+@Data
+public class RecordMetaDataInterval implements RecordMetaData {
+
+    private final long from;
+    private final long to;
+    private final URI uri;
+    private Class<?> readerClass;
+
+    public RecordMetaDataInterval(long from, long to, URI uri) {
+        this(from, to, uri, null);
+    }
+
+    @Override
+    public String getLocation() {
+        return "interval(" + from + "," + to + ")";
+    }
+
+    @Override
+    public URI getURI() {
+        return uri;
+    }
+
+    @Override
+    public Class<?> getReaderClass() {
+        return readerClass;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataLine.java

@@ -0,0 +1,59 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.metadata;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import org.apache.commons.io.FilenameUtils;
+
+import java.net.URI;
+
+@AllArgsConstructor
+@Data
+public class RecordMetaDataLine implements RecordMetaData {
+
+    private int lineNumber;
+    private URI uri;
+    private Class<?> readerClass;
+
+
+    @Override
+    public String getLocation() {
+        String filename;
+        if (uri != null) {
+            String str = uri.toString();
+            filename = FilenameUtils.getBaseName(str) + "." + FilenameUtils.getExtension(str) + " ";
+        } else {
+            filename = "";
+        }
+        return filename + "line " + lineNumber;
+    }
+
+    @Override
+    public URI getURI() {
+        return uri;
+    }
+
+    @Override
+    public Class<?> getReaderClass() {
+        return readerClass;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataLineInterval.java

@@ -0,0 +1,60 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.metadata;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import org.apache.commons.io.FilenameUtils;
+
+import java.net.URI;
+
+@AllArgsConstructor
+@Data
+public class RecordMetaDataLineInterval implements RecordMetaData {
+
+    private int lineNumberStart;
+    private int lineNumberEnd;
+    private URI uri;
+    private Class<?> readerClass;
+
+
+    @Override
+    public String getLocation() {
+        String filename;
+        if (uri != null) {
+            String str = uri.toString();
+            filename = FilenameUtils.getBaseName(str) + "." + FilenameUtils.getExtension(str) + " ";
+        } else {
+            filename = "";
+        }
+        return filename + "lines " + lineNumberStart + "-" + lineNumberEnd;
+    }
+
+    @Override
+    public URI getURI() {
+        return uri;
+    }
+
+    @Override
+    public Class<?> getReaderClass() {
+        return readerClass;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataURI.java

@@ -0,0 +1,53 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.metadata;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.net.URI;
+
+@AllArgsConstructor
+@Data
+public class RecordMetaDataURI implements RecordMetaData {
+
+    private final URI uri;
+    private final Class<?> readerClass;
+
+    public RecordMetaDataURI(URI uri) {
+        this(uri, null);
+    }
+
+    @Override
+    public String getLocation() {
+        return uri.toString();
+    }
+
+    @Override
+    public URI getURI() {
+        return uri;
+    }
+
+    @Override
+    public Class<?> getReaderClass() {
+        return readerClass;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/BaseRecordReader.java

@@ -0,0 +1,87 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader;
+
+import org.datavec.api.records.listener.RecordListener;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.split.StreamInputSplit;
+import org.datavec.api.split.streams.FileStreamCreatorFunction;
+import org.datavec.api.writable.Writable;
+import org.nd4j.common.function.Function;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+public abstract class BaseRecordReader implements RecordReader {
+
+    protected InputSplit inputSplit;
+    protected List<RecordListener> listeners = new ArrayList<>();
+    protected Function<URI,InputStream> streamCreatorFn = new FileStreamCreatorFunction();
+
+    /** Invokes {@link RecordListener#recordRead(RecordReader, Object)} on all listeners. */
+    protected void invokeListeners(Object record) {
+        for (RecordListener listener : listeners) {
+            listener.recordRead(this, record);
+        }
+    }
+
+    @Override
+    public void initialize(InputSplit split) throws IOException, InterruptedException {
+        this.inputSplit = split;
+        if(split instanceof StreamInputSplit){
+            StreamInputSplit s = (StreamInputSplit)split;
+            if(s.getStreamCreatorFn() != null){
+                this.streamCreatorFn = s.getStreamCreatorFn();
+            }
+        }
+    }
+
+    @Override
+    public List<RecordListener> getListeners() {
+        return listeners;
+    }
+
+    @Override
+    public void setListeners(Collection<RecordListener> listeners) {
+        this.listeners = (listeners instanceof List ? (List<RecordListener>) listeners : new ArrayList<>(listeners));
+    }
+
+    @Override
+    public void setListeners(RecordListener... listeners) {
+        setListeners(Arrays.asList(listeners));
+    }
+
+
+    @Override
+    public boolean batchesSupported() {
+        return false;
+    }
+
+    @Override
+    public List<List<Writable>> next(int num) {
+        throw new UnsupportedOperationException();
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/RecordReader.java

@@ -0,0 +1,168 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader;
+
+import org.datavec.api.conf.Configurable;
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.listener.RecordListener;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Writable;
+
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URI;
+import java.util.Collection;
+import java.util.List;
+
+public interface RecordReader extends Closeable, Serializable, Configurable {
+
+    String NAME_SPACE = RecordReader.class.getName();
+
+    String APPEND_LABEL = NAME_SPACE + ".appendlabel";
+    String LABELS = NAME_SPACE + ".labels";
+
+    /**
+     * Called once at initialization.
+     *
+     * @param split the split that defines the range of records to read
+     * @throws java.io.IOException
+     * @throws InterruptedException
+     */
+    void initialize(InputSplit split) throws IOException, InterruptedException;
+
+    /**
+     * Called once at initialization.
+     *
+     * @param conf  a configuration for initialization
+     * @param split the split that defines the range of records to read
+     * @throws java.io.IOException
+     * @throws InterruptedException
+     */
+    void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException;
+
+    /**
+     * This method returns true, if next(int) signature is supported by this RecordReader implementation.
+     *
+     * @return
+     */
+    boolean batchesSupported();
+
+    /**
+     * This method will be used, if batchesSupported() returns true.
+     *
+     * @param num
+     * @return
+     */
+    List<List<Writable>> next(int num);
+
+    /**
+     * Get the next record
+     *
+     * @return
+     */
+    List<Writable> next();
+
+
+
+    /**
+     * Whether there are anymore records
+     *
+     * @return
+     */
+    boolean hasNext();
+
+    /**
+     * List of label strings
+     *
+     * @return
+     */
+    List<String> getLabels();
+
+    /**
+     * Reset record reader iterator
+     *
+     * @return
+     */
+    void reset();
+
+    /**
+     * @return True if the record reader can be reset, false otherwise. Note that some record readers cannot be reset -
+     * for example, if they are backed by a non-resettable input split (such as certain types of streams)
+     */
+    boolean resetSupported();
+
+    /**
+     * Load the record from the given DataInputStream
+     * Unlike {@link #next()} the internal state of the RecordReader is not modified
+     * Implementations of this method should not close the DataInputStream
+     *
+     * @throws IOException if error occurs during reading from the input stream
+     */
+    List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException;
+
+
+    /**
+     * Similar to {@link #next()}, but returns a {@link Record} object, that may include metadata such as the source
+     * of the data
+     *
+     * @return next record
+     */
+    Record nextRecord();
+
+    /**
+     * Load a single record from the given {@link RecordMetaData} instance<br>
+     * Note: that for data that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to
+     * load multiple records at once using {@link #loadFromMetaData(List)}
+     *
+     * @param recordMetaData Metadata for the record that we want to load from
+     * @return Single record for the given RecordMetaData instance
+     * @throws IOException If I/O error occurs during loading
+     */
+    Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException;
+
+    /**
+     * Load multiple records from the given a list of {@link RecordMetaData} instances<br>
+     *
+     * @param recordMetaDatas Metadata for the records that we want to load from
+     * @return Multiple records for the given RecordMetaData instances
+     * @throws IOException If I/O error occurs during loading
+     */
+    List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException;
+
+    /**
+     * Get the record listeners for this record reader.
+     */
+    List<RecordListener> getListeners();
+
+    /**
+     * Set the record listeners for this record reader.
+     */
+    void setListeners(RecordListener... listeners);
+
+    /**
+     * Set the record listeners for this record reader.
+     */
+    void setListeners(Collection<RecordListener> listeners);
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/SequenceRecordReader.java

@@ -0,0 +1,77 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader;
+
+import org.datavec.api.records.Record;
+import org.datavec.api.records.SequenceRecord;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+
+public interface SequenceRecordReader extends RecordReader {
+    /**
+     * Returns a sequence record.
+     *
+     * @return a sequence of records
+     */
+    List<List<Writable>> sequenceRecord();
+
+    /**
+     * Load a sequence record from the given DataInputStream
+     * Unlike {@link #next()} the internal state of the RecordReader is not modified
+     * Implementations of this method should not close the DataInputStream
+     *
+     * @throws IOException if error occurs during reading from the input stream
+     */
+    List<List<Writable>> sequenceRecord(URI uri, DataInputStream dataInputStream) throws IOException;
+
+    /**
+     * Similar to {@link #sequenceRecord()}, but returns a {@link Record} object, that may include metadata such as the source
+     * of the data
+     *
+     * @return next sequence record
+     */
+    SequenceRecord nextSequence();
+
+    /**
+     * Load a single sequence record from the given {@link RecordMetaData} instance<br>
+     * Note: that for data that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to
+     * load multiple records at once using {@link #loadSequenceFromMetaData(List)}
+     *
+     * @param recordMetaData Metadata for the sequence record that we want to load from
+     * @return Single sequence record for the given RecordMetaData instance
+     * @throws IOException If I/O error occurs during loading
+     */
+    SequenceRecord loadSequenceFromMetaData(RecordMetaData recordMetaData) throws IOException;
+
+    /**
+     * Load multiple sequence records from the given a list of {@link RecordMetaData} instances<br>
+     *
+     * @param recordMetaDatas Metadata for the records that we want to load from
+     * @return Multiple sequence record for the given RecordMetaData instances
+     * @throws IOException If I/O error occurs during loading
+     */
+    List<SequenceRecord> loadSequenceFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException;
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/factory/RecordReaderFactory.java

@@ -0,0 +1,37 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.factory;
+
+import org.datavec.api.exceptions.UnknownFormatException;
+import org.datavec.api.records.reader.RecordReader;
+
+import java.net.URI;
+
+public interface RecordReaderFactory {
+    /**
+     * Creates instance of RecordReader
+     *
+     * @param uri
+     * @return record reader instance
+     * @throws UnknownFormatException
+     */
+    RecordReader create(URI uri) throws UnknownFormatException;
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/factory/RecordWriterFactory.java

@@ -0,0 +1,37 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.factory;
+
+import org.datavec.api.records.writer.RecordWriter;
+
+import java.net.URI;
+
+public interface RecordWriterFactory {
+
+    /**
+     *
+     * @param uri destination for saving model
+     * @return record writer instance
+     * @throws Exception
+     */
+
+    RecordWriter create(URI uri) throws Exception;
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/ComposableRecordReader.java

@@ -0,0 +1,144 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl;
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.reader.BaseRecordReader;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author sonali
+ */
+public class ComposableRecordReader extends BaseRecordReader {
+
+    private RecordReader[] readers;
+
+    public ComposableRecordReader(RecordReader... readers) {
+        this.readers = readers;
+    }
+
+    @Override
+    public void initialize(InputSplit split) throws IOException, InterruptedException {
+
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+
+    }
+
+    @Override
+    public List<Writable> next() {
+        List<Writable> ret = new ArrayList<>();
+        if (this.hasNext()) {
+            for (RecordReader reader : readers) {
+                ret.addAll(reader.next());
+            }
+        }
+        invokeListeners(ret);
+        return ret;
+    }
+
+    @Override
+    public boolean hasNext() {
+        boolean readersHasNext = true;
+        for (RecordReader reader : readers) {
+            readersHasNext = readersHasNext && reader.hasNext();
+        }
+        return readersHasNext;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+    @Override
+    public void close() throws IOException {
+        for (RecordReader reader : readers)
+            reader.close();
+    }
+
+    @Override
+    public void setConf(Configuration conf) {
+        for (RecordReader reader : readers) {
+            reader.setConf(conf);
+        }
+    }
+
+    @Override
+    public Configuration getConf() {
+        for (RecordReader reader : readers) {
+            return reader.getConf();
+        }
+        return null;
+    }
+
+    @Override
+    public void reset() {
+        for (RecordReader reader : readers)
+            reader.reset();
+
+    }
+
+    @Override
+    public boolean resetSupported() {
+        for(RecordReader rr : readers){
+            if(!rr.resetSupported()){
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        throw new UnsupportedOperationException(
+                        "Generating records from DataInputStream not supported for ComposableRecordReader");
+    }
+
+    @Override
+    public Record nextRecord() {
+        return new org.datavec.api.records.impl.Record(next(), null);
+    }
+
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        throw new UnsupportedOperationException("Loading from metadata not yet implemented");
+    }
+
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        throw new UnsupportedOperationException("Loading from metadata not yet implemented");
+    }
+
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/ConcatenatingRecordReader.java

@@ -0,0 +1,138 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl;
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.reader.BaseRecordReader;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+
+public class ConcatenatingRecordReader extends BaseRecordReader {
+
+    private RecordReader[] readers;
+
+    public ConcatenatingRecordReader(RecordReader... readers) {
+        this.readers = readers;
+    }
+
+    @Override
+    public void initialize(InputSplit split) throws IOException, InterruptedException {
+
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+
+    }
+
+    @Override
+    public List<Writable> next() {
+        List<Writable> out = null;
+        for( RecordReader rr : readers){
+            if(rr.hasNext()){
+                out = rr.next();
+                break;
+            }
+        }
+        invokeListeners(out);
+        return out;
+    }
+
+    @Override
+    public boolean hasNext() {
+        for (RecordReader reader : readers) {
+            if(reader.hasNext()){
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+    @Override
+    public void close() throws IOException {
+        for (RecordReader reader : readers)
+            reader.close();
+    }
+
+    @Override
+    public void setConf(Configuration conf) {
+        for (RecordReader reader : readers) {
+            reader.setConf(conf);
+        }
+    }
+
+    @Override
+    public Configuration getConf() {
+        return readers[0].getConf();
+    }
+
+    @Override
+    public void reset() {
+        for (RecordReader reader : readers)
+            reader.reset();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        for(RecordReader rr : readers){
+            if(!rr.resetSupported()){
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        throw new UnsupportedOperationException(
+                        "Generating records from DataInputStream not supported for ComposableRecordReader");
+    }
+
+    @Override
+    public Record nextRecord() {
+        return new org.datavec.api.records.impl.Record(next(), null);
+    }
+
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        throw new UnsupportedOperationException("Loading from metadata not yet implemented");
+    }
+
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        throw new UnsupportedOperationException("Loading from metadata not yet implemented");
+    }
+
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/FileRecordReader.java

@@ -0,0 +1,241 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataURI;
+import org.datavec.api.records.reader.BaseRecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.IntWritable;
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+
+import java.io.*;
+import java.net.URI;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+
+/**
+ * File reader/writer
+ *
+ * @author Adam Gibson
+ */
+public class FileRecordReader extends BaseRecordReader {
+
+    protected Iterator<URI> locationsIterator;
+    protected Configuration conf;
+    protected URI currentUri;
+    protected List<String> labels;
+    protected boolean appendLabel = false;
+    @Getter @Setter
+    protected String charset = StandardCharsets.UTF_8.name(); //Using String as StandardCharsets.UTF_8 is not serializable
+
+    public FileRecordReader() {}
+
+    @Override
+    public void initialize(InputSplit split) throws IOException, InterruptedException {
+        super.initialize(split);
+        doInitialize(split);
+    }
+
+
+    protected void doInitialize(InputSplit split) {
+
+        if (labels == null && appendLabel) {
+            URI[] locations = split.locations();
+            if (locations.length > 0) {
+                Set<String> labels = new HashSet<>();
+                for(URI u : locations){
+                    String[] pathSplit = u.toString().split("[/\\\\]");
+                    labels.add(pathSplit[pathSplit.length-2]);
+                }
+                this.labels = new ArrayList<>(labels);
+                Collections.sort(this.labels);
+            }
+        }
+        locationsIterator = split.locationsIterator();
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        appendLabel = conf.getBoolean(APPEND_LABEL, true);
+        doInitialize(split);
+        this.inputSplit = split;
+        this.conf = conf;
+    }
+
+    @Override
+    public List<Writable> next() {
+        return nextRecord().getRecord();
+    }
+
+    private List<Writable> loadFromStream(URI uri, InputStream next, Charset charset) {
+        List<Writable> ret = new ArrayList<>();
+        try {
+            if(!(next instanceof BufferedInputStream)){
+                next = new BufferedInputStream(next);
+            }
+            String s = org.apache.commons.io.IOUtils.toString(next, charset);
+            ret.add(new Text(s));
+            if (appendLabel) {
+                int idx = getLabel(uri);
+                ret.add(new IntWritable(idx));
+            }
+        } catch (IOException e) {
+            throw new IllegalStateException("Error reading from input stream: " + uri);
+        }
+        return ret;
+    }
+
+    /**
+     * Return the current label.
+     * The index of the current file's parent directory
+     * in the label list
+     * @return The index of the current file's parent directory
+     */
+    public int getCurrentLabel() {
+        return getLabel(currentUri);
+    }
+
+    public int getLabel(URI uri){
+        String s = uri.toString();
+        int lastIdx = Math.max(s.lastIndexOf('/'), s.lastIndexOf('\\'));    //Note: if neither are found, -1 is fine here
+        String sub = s.substring(0, lastIdx);
+        int secondLastIdx = Math.max(sub.lastIndexOf('/'), sub.lastIndexOf('\\'));
+        String name = s.substring(secondLastIdx+1, lastIdx);
+        return labels.indexOf(name);
+    }
+
+    public List<String> getLabels() {
+        return labels;
+    }
+
+    public void setLabels(List<String> labels) {
+        this.labels = labels;
+    }
+
+    @Override
+    public boolean hasNext() {
+        return locationsIterator.hasNext();
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+
+    @Override
+    public void setConf(Configuration conf) {
+        this.conf = conf;
+    }
+
+    @Override
+    public Configuration getConf() {
+        return conf;
+    }
+
+    @Override
+    public List<List<Writable>> next(int num) {
+        List<List<Writable>> ret = new ArrayList<>(num);
+        int numBatches = 0;
+        while (hasNext() && numBatches < num) {
+            ret.add(next());
+        }
+
+        return ret;
+    }
+    @Override
+    public void reset() {
+        if (inputSplit == null)
+            throw new UnsupportedOperationException("Cannot reset without first initializing");
+        try {
+            doInitialize(inputSplit);
+        } catch (Exception e) {
+            throw new RuntimeException("Error during LineRecordReader reset", e);
+        }
+    }
+
+    @Override
+    public boolean resetSupported() {
+        if(inputSplit != null){
+            return inputSplit.resetSupported();
+        }
+        return false;   //reset() throws exception on reset() if inputSplit is null
+    }
+
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        invokeListeners(uri);
+        //Here: reading the entire file to a Text writable
+        BufferedReader br = new BufferedReader(new InputStreamReader(dataInputStream));
+        StringBuilder sb = new StringBuilder();
+        String line;
+        while ((line = br.readLine()) != null) {
+            sb.append(line).append("\n");
+        }
+        return Collections.singletonList((Writable) new Text(sb.toString()));
+    }
+
+    @Override
+    public Record nextRecord() {
+        URI next = locationsIterator.next();
+        invokeListeners(next);
+
+        List<Writable> ret;
+        try(InputStream s = streamCreatorFn.apply(next)) {
+            ret = loadFromStream(next, s, Charset.forName(charset));
+        } catch (IOException e){
+            throw new RuntimeException("Error reading from stream for URI: " + next);
+        }
+
+        return new org.datavec.api.records.impl.Record(ret,new RecordMetaDataURI(next, FileRecordReader.class));
+    }
+
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return loadFromMetaData(Collections.singletonList(recordMetaData)).get(0);
+    }
+
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        List<Record> out = new ArrayList<>();
+
+        for (RecordMetaData meta : recordMetaDatas) {
+            URI uri = meta.getURI();
+
+            List<Writable> list;
+            try(InputStream s = streamCreatorFn.apply(uri)) {
+                list = loadFromStream(uri, s, Charset.forName(charset));
+            } catch (IOException e){
+                throw new RuntimeException("Error reading from stream for URI: " + uri);
+            }
+
+            out.add(new org.datavec.api.records.impl.Record(list, meta));
+        }
+
+        return out;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/LineRecordReader.java

@@ -0,0 +1,374 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.LineIterator;
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataLine;
+import org.datavec.api.records.reader.BaseRecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.split.InputStreamInputSplit;
+import org.datavec.api.split.StringSplit;
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+import org.nd4j.common.base.Preconditions;
+import org.nd4j.common.primitives.Triple;
+
+import java.io.*;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+
+/**
+ * Reads files line by line
+ *
+ * @author Adam Gibson
+ */
+@Slf4j
+public class LineRecordReader extends BaseRecordReader {
+
+
+    private Iterator<String> iter;
+    protected URI[] locations;
+    protected int splitIndex = 0;
+    protected int lineIndex = 0; //Line index within the current split
+    protected Configuration conf;
+    protected boolean initialized;
+    @Getter @Setter
+    protected String charset = StandardCharsets.UTF_8.name(); //Using String as StandardCharsets.UTF_8 is not serializable
+
+    @Override
+    public void initialize(InputSplit split) throws IOException, InterruptedException {
+        super.initialize(split);
+        if(!(inputSplit instanceof StringSplit || inputSplit instanceof InputStreamInputSplit)){
+            final ArrayList<URI> uris = new ArrayList<>();
+            final Iterator<URI> uriIterator = inputSplit.locationsIterator();
+            while(uriIterator.hasNext()) uris.add(uriIterator.next());
+
+            this.locations = uris.toArray(new URI[0]);
+        }
+        this.iter = getIterator(0);
+        this.initialized = true;
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        this.conf = conf;
+        initialize(split);
+    }
+
+    @Override
+    public List<Writable> next() {
+        Preconditions.checkState(initialized, "Record reader has not been initialized");
+        List<Writable> ret = new ArrayList<>();
+
+        if (iter.hasNext()) {
+            String record = iter.next();
+            invokeListeners(record);
+            ret.add(new Text(record));
+            lineIndex++;
+            return ret;
+        } else {
+            if (!(inputSplit instanceof StringSplit) && splitIndex < locations.length - 1) {
+                splitIndex++;
+                lineIndex = 0; //New split opened -> reset line index
+                try {
+                    close();
+                    iter = getIterator(splitIndex);
+                    onLocationOpen(locations[splitIndex]);
+                } catch (IOException e) {
+                    log.error("",e);
+                }
+
+                if (iter.hasNext()) {
+                    String record = iter.next();
+                    invokeListeners(record);
+                    ret.add(new Text(record));
+                    lineIndex++;
+                    return ret;
+                }
+            }
+
+            throw new NoSuchElementException("No more elements found!");
+        }
+    }
+
+    @Override
+    public boolean hasNext() {
+        Preconditions.checkState(initialized, "Record reader has not been initialized");
+
+        if (iter != null && iter.hasNext()) {
+            return true;
+        } else {
+            if (locations != null && !(inputSplit instanceof StringSplit) && splitIndex < locations.length - 1) {
+                splitIndex++;
+                lineIndex = 0; //New split -> reset line count
+                try {
+                    close();
+                    iter = getIterator(splitIndex);
+                    onLocationOpen(locations[splitIndex]);
+                } catch (IOException e) {
+                    log.error("",e);
+                }
+
+                return iter.hasNext();
+            }
+
+            return false;
+        }
+    }
+
+    protected void onLocationOpen(URI location) {
+
+    }
+
+    @Override
+    public void close() throws IOException {
+        if (iter != null) {
+            if (iter instanceof LineIterator) {
+                LineIterator iter2 = (LineIterator) iter;
+                iter2.close();
+            }
+        }
+    }
+
+    @Override
+    public void setConf(Configuration conf) {
+        this.conf = conf;
+    }
+
+    @Override
+    public Configuration getConf() {
+        return conf;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+    @Override
+    public void reset() {
+        if (inputSplit == null)
+            throw new UnsupportedOperationException("Cannot reset without first initializing");
+        try {
+            inputSplit.reset();
+            close();
+            initialize(inputSplit);
+            splitIndex = 0;
+        } catch (Exception e) {
+            throw new RuntimeException("Error during LineRecordReader reset", e);
+        }
+        lineIndex = 0;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        if(inputSplit != null){
+            return inputSplit.resetSupported();
+        }
+        return true;
+    }
+
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        invokeListeners(uri);
+        //Here: we are reading a single line from the DataInputStream
+        BufferedReader br = new BufferedReader(new InputStreamReader(dataInputStream));
+        String line = br.readLine();
+        return Collections.singletonList((Writable) new Text(line));
+    }
+
+    protected Iterator<String> getIterator(int location) {
+        Iterator<String> iterator = null;
+        if (inputSplit instanceof StringSplit) {
+            StringSplit stringSplit = (StringSplit) inputSplit;
+            iterator = Collections.singletonList(stringSplit.getData()).listIterator();
+        } else if (inputSplit instanceof InputStreamInputSplit) {
+            InputStream is = ((InputStreamInputSplit) inputSplit).getIs();
+            if (is != null) {
+                try {
+                    iterator = IOUtils.lineIterator(new InputStreamReader(is, charset));
+                } catch (UnsupportedEncodingException e){
+                    throw new RuntimeException("Unsupported encoding: " + charset, e);
+                }
+            }
+        } else {
+            if (locations.length > 0) {
+                InputStream inputStream = streamCreatorFn.apply(locations[location]);
+                try {
+                    iterator = IOUtils.lineIterator(new InputStreamReader(inputStream, charset));
+                } catch (UnsupportedEncodingException e){
+                    throw new RuntimeException("Unsupported encoding: " + charset, e);
+                }
+            }
+        }
+        if (iterator == null)
+            throw new UnsupportedOperationException("Unknown input split: " + inputSplit);
+        return iterator;
+    }
+
+    @SneakyThrows
+    protected void closeIfRequired(Iterator<String> iterator) {
+        if (iterator instanceof LineIterator) {
+            LineIterator iter = (LineIterator) iterator;
+            iter.close();
+        }
+    }
+
+    @Override
+    public Record nextRecord() {
+        List<Writable> next = next();
+        URI uri = (locations == null || locations.length < 1 ? null : locations[splitIndex]);
+        RecordMetaData meta = new RecordMetaDataLine(this.lineIndex - 1, uri, LineRecordReader.class); //-1 as line number has been incremented already...
+        return new org.datavec.api.records.impl.Record(next, meta);
+    }
+
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return null;
+    }
+
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        //First: create a sorted list of the RecordMetaData
+        List<Triple<Integer, RecordMetaDataLine, List<Writable>>> list = new ArrayList<>();
+        Set<URI> uris = new HashSet<>();
+        Iterator<RecordMetaData> iter = recordMetaDatas.iterator();
+        int count = 0;
+        while (iter.hasNext()) {
+            RecordMetaData rmd = iter.next();
+            if (!(rmd instanceof RecordMetaDataLine)) {
+                throw new IllegalArgumentException(
+                                "Invalid metadata; expected RecordMetaDataLine instance; got: " + rmd);
+            }
+            list.add(new Triple<>(count++, (RecordMetaDataLine) rmd, (List<Writable>) null));
+            if (rmd.getURI() != null)
+                uris.add(rmd.getURI());
+        }
+        List<URI> sortedURIs = null;
+        if (uris.size() > 0) {
+            sortedURIs = new ArrayList<>(uris);
+            Collections.sort(sortedURIs);
+        }
+
+        //Sort by URI first (if possible - don't always have URIs though, for String split etc), then sort by line number:
+        Collections.sort(list, new Comparator<Triple<Integer, RecordMetaDataLine, List<Writable>>>() {
+            @Override
+            public int compare(Triple<Integer, RecordMetaDataLine, List<Writable>> o1,
+                            Triple<Integer, RecordMetaDataLine, List<Writable>> o2) {
+                if (o1.getSecond().getURI() != null) {
+                    if (!o1.getSecond().getURI().equals(o2.getSecond().getURI())) {
+                        return o1.getSecond().getURI().compareTo(o2.getSecond().getURI());
+                    }
+                }
+                return Integer.compare(o1.getSecond().getLineNumber(), o2.getSecond().getLineNumber());
+            }
+        });
+
+        if (uris.size() > 0 && sortedURIs != null) {
+            //URIs case - possibly with multiple URIs
+            Iterator<Triple<Integer, RecordMetaDataLine, List<Writable>>> metaIter = list.iterator(); //Currently sorted by URI, then line number
+
+            URI currentURI = sortedURIs.get(0);
+            Iterator<String> currentUriIter = IOUtils.lineIterator(streamCreatorFn.apply(currentURI), charset);
+
+            int currentURIIdx = 0; //Index of URI
+            int currentLineIdx = 0; //Index of the line for the current URI
+            String line = currentUriIter.next();
+            while (metaIter.hasNext()) {
+                Triple<Integer, RecordMetaDataLine, List<Writable>> t = metaIter.next();
+                URI thisURI = t.getSecond().getURI();
+                int nextLineIdx = t.getSecond().getLineNumber();
+
+                //First: find the right URI for this record...
+                while (!currentURI.equals(thisURI)) {
+                    //Iterate to the next URI
+                    currentURIIdx++;
+                    if (currentURIIdx >= sortedURIs.size()) {
+                        //Should never happen
+                        throw new IllegalStateException(
+                                        "Count not find URI " + thisURI + " in URIs list: " + sortedURIs);
+                    }
+                    currentURI = sortedURIs.get(currentURIIdx);
+                    currentLineIdx = 0;
+                    if (currentURI.equals(thisURI)) {
+                        //Found the correct URI for this MetaData instance
+                        closeIfRequired(currentUriIter);
+                        currentUriIter = IOUtils.lineIterator(new InputStreamReader(currentURI.toURL().openStream()));
+                        line = currentUriIter.next();
+                    }
+                }
+
+                //Have the correct URI/iter open -> scan to the required line
+                while (currentLineIdx < nextLineIdx && currentUriIter.hasNext()) {
+                    line = currentUriIter.next();
+                    currentLineIdx++;
+                }
+                if (currentLineIdx < nextLineIdx && !currentUriIter.hasNext()) {
+                    throw new IllegalStateException("Could not get line " + nextLineIdx + " from URI " + currentURI
+                                    + ": has only " + currentLineIdx + " lines");
+                }
+                t.setThird(Collections.<Writable>singletonList(new Text(line)));
+            }
+        } else {
+            //Not URI based: String split, etc
+            Iterator<String> iterator = getIterator(0);
+            Iterator<Triple<Integer, RecordMetaDataLine, List<Writable>>> metaIter = list.iterator();
+            int currentLineIdx = 0;
+            String line = iterator.next();
+            while (metaIter.hasNext()) {
+                Triple<Integer, RecordMetaDataLine, List<Writable>> t = metaIter.next();
+                int nextLineIdx = t.getSecond().getLineNumber();
+                while (currentLineIdx < nextLineIdx && iterator.hasNext()) {
+                    line = iterator.next();
+                    currentLineIdx++;
+                }
+                t.setThird(Collections.<Writable>singletonList(new Text(line)));
+            }
+            closeIfRequired(iterator);
+        }
+
+
+        //Now, sort by the original (request) order:
+        Collections.sort(list, new Comparator<Triple<Integer, RecordMetaDataLine, List<Writable>>>() {
+            @Override
+            public int compare(Triple<Integer, RecordMetaDataLine, List<Writable>> o1,
+                            Triple<Integer, RecordMetaDataLine, List<Writable>> o2) {
+                return Integer.compare(o1.getFirst(), o2.getFirst());
+            }
+        });
+
+        //And return...
+        List<Record> out = new ArrayList<>();
+        for (Triple<Integer, RecordMetaDataLine, List<Writable>> t : list) {
+            out.add(new org.datavec.api.records.impl.Record(t.getThird(), t.getSecond()));
+        }
+        return out;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionRecordReader.java

@@ -0,0 +1,160 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.collection;
+
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataIndex;
+import org.datavec.api.records.reader.BaseRecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.*;
+
+public class CollectionRecordReader extends BaseRecordReader {
+    private Iterator<? extends Collection<Writable>> records;
+    private final Collection<? extends Collection<Writable>> original;
+    private int count = 0;
+
+    public CollectionRecordReader(Collection<? extends Collection<Writable>> records) {
+        this.records = records.iterator();
+        this.original = records;
+    }
+
+    @Override
+    public void initialize(InputSplit split) throws IOException, InterruptedException {
+
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        initialize(split);
+    }
+
+    @Override
+    public List<Writable> next() {
+        Collection<Writable> next = records.next();
+        List<Writable> record = (next instanceof List ? (List<Writable>) next : new ArrayList<>(next));
+        invokeListeners(record);
+        count++;
+        return record;
+    }
+
+    @Override
+    public boolean hasNext() {
+        return records.hasNext();
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+
+    @Override
+    public void setConf(Configuration conf) {
+
+    }
+
+    @Override
+    public Configuration getConf() {
+        return null;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+    @Override
+    public void reset() {
+        this.records = original.iterator();
+        this.count = 0;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        throw new UnsupportedOperationException(
+                        "Generating records from DataInputStream not supported for CollectionRecordReader");
+    }
+
+
+    @Override
+    public Record nextRecord() {
+        return new org.datavec.api.records.impl.Record(next(),
+                        new RecordMetaDataIndex(count - 1, null, CollectionRecordReader.class));
+    }
+
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return loadFromMetaData(Collections.singletonList(recordMetaData)).get(0);
+    }
+
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        Set<Integer> toLoad = new HashSet<>();
+        for (RecordMetaData recordMetaData : recordMetaDatas) {
+            if (!(recordMetaData instanceof RecordMetaDataIndex)) {
+                throw new IllegalArgumentException("Expected RecordMetaDataIndex; got: " + recordMetaData);
+            }
+            long idx = ((RecordMetaDataIndex) recordMetaData).getIndex();
+            if (idx >= original.size()) {
+                throw new IllegalStateException(
+                                "Cannot get index " + idx + " from collection: contains " + original + " elements");
+            }
+            toLoad.add((int) idx);
+        }
+
+        List<Record> out = new ArrayList<>();
+        if (original instanceof List) {
+            List<Collection<Writable>> asList = (List<Collection<Writable>>) original;
+            for (Integer i : toLoad) {
+                List<Writable> l = new ArrayList<>(asList.get(i));
+                Record r = new org.datavec.api.records.impl.Record(l,
+                                new RecordMetaDataIndex(i, null, CollectionRecordReader.class));
+                out.add(r);
+            }
+        } else {
+            Iterator<? extends Collection<Writable>> iter = original.iterator();
+            int i = 0;
+            while (iter.hasNext()) {
+                Collection<Writable> c = iter.next();
+                if (!toLoad.contains(i++)) {
+                    continue;
+                }
+                List<Writable> l = (c instanceof List ? ((List<Writable>) c) : new ArrayList<>(c));
+                Record r = new org.datavec.api.records.impl.Record(l,
+                                new RecordMetaDataIndex(i - 1, null, CollectionRecordReader.class));
+                out.add(r);
+            }
+        }
+        return out;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionSequenceRecordReader.java

@@ -0,0 +1,192 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.collection;
+
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.SequenceRecord;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataIndex;
+import org.datavec.api.records.reader.BaseRecordReader;
+import org.datavec.api.records.reader.SequenceRecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.*;
+
+public class CollectionSequenceRecordReader extends BaseRecordReader implements SequenceRecordReader {
+    private Iterator<? extends Collection<? extends Collection<Writable>>> records;
+    private final Collection<? extends Collection<? extends Collection<Writable>>> original;
+    private int count = 0;
+
+    /**
+     *
+     * @param records    Collection of sequences. For example, List<List<List<Writable>>> where the inner  two lists
+     *                   are a sequence, and the outer list/collection is a list of sequences
+     */
+    public CollectionSequenceRecordReader(Collection<? extends Collection<? extends Collection<Writable>>> records) {
+        this.records = records.iterator();
+        this.original = records;
+    }
+
+    @Override
+    public void initialize(InputSplit split) throws IOException, InterruptedException {
+
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        initialize(split);
+    }
+
+    @Override
+    public List<Writable> next() {
+        throw new UnsupportedOperationException(
+                        "next() not supported for CollectionSequencRecordReader; use sequenceRecord()");
+    }
+
+    @Override
+    public boolean hasNext() {
+        return records.hasNext();
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+
+    @Override
+    public void setConf(Configuration conf) {
+
+    }
+
+    @Override
+    public Configuration getConf() {
+        return null;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+    @Override
+    public void reset() {
+        this.records = original.iterator();
+        this.count = 0;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        throw new UnsupportedOperationException(
+                        "Generating records from DataInputStream not supported for SequenceCollectionRecordReader");
+    }
+
+    @Override
+    public Record nextRecord() {
+        return new org.datavec.api.records.impl.Record(next(), null);
+    }
+
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        throw new UnsupportedOperationException("Loading from metadata not yet implemented");
+    }
+
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        throw new UnsupportedOperationException("Loading from metadata not yet implemented");
+    }
+
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public List<List<Writable>> sequenceRecord() {
+        List<List<Writable>> record = toList(records.next());
+        invokeListeners(record);
+        count++;
+        return record;
+    }
+
+    @Override
+    public List<List<Writable>> sequenceRecord(URI uri, DataInputStream dataInputStream) throws IOException {
+        throw new UnsupportedOperationException(
+                        "Generating records from DataInputStream not supported for SequenceCollectionRecordReader");
+    }
+
+    @Override
+    public SequenceRecord nextSequence() {
+        return new org.datavec.api.records.impl.SequenceRecord(sequenceRecord(),
+                        new RecordMetaDataIndex(count - 1, null, CollectionSequenceRecordReader.class));
+    }
+
+    @Override
+    public SequenceRecord loadSequenceFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return loadSequenceFromMetaData(Collections.singletonList(recordMetaData)).get(0);
+    }
+
+    @Override
+    public List<SequenceRecord> loadSequenceFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        Set<Integer> toLoad = new LinkedHashSet<>();
+        for (RecordMetaData recordMetaData : recordMetaDatas) {
+            if (!(recordMetaData instanceof RecordMetaDataIndex)) {
+                throw new IllegalArgumentException("Expected RecordMetaDataIndex; got: " + recordMetaData);
+            }
+            long idx = ((RecordMetaDataIndex) recordMetaData).getIndex();
+            if (idx >= original.size()) {
+                throw new IllegalStateException(
+                                "Cannot get index " + idx + " from collection: contains " + original + " elements");
+            }
+            toLoad.add((int) idx);
+        }
+
+        List<SequenceRecord> out = new ArrayList<>();
+        Iterator<? extends Collection<? extends Collection<Writable>>> iter = original.iterator();
+        int i = 0;
+        while (iter.hasNext()) {
+            Collection<? extends Collection<Writable>> c = iter.next();
+            if (!toLoad.contains(i++)) {
+                continue;
+            }
+            List<List<Writable>> record = toList(c);
+            SequenceRecord r = new org.datavec.api.records.impl.SequenceRecord(record,
+                            new RecordMetaDataIndex(i - 1, null, CollectionSequenceRecordReader.class));
+            out.add(r);
+        }
+        return out;
+    }
+
+    private static List<List<Writable>> toList(Collection<? extends Collection<Writable>> next) {
+        List<List<Writable>> record = new ArrayList<>();
+        for (Collection<Writable> c : next) {
+            record.add(new ArrayList<>(c));
+        }
+        return record;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/ListStringRecordReader.java

@@ -0,0 +1,188 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.collection;
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.reader.BaseRecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.split.ListStringSplit;
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class ListStringRecordReader extends BaseRecordReader {
+    private List<List<String>> delimitedData;
+    private Iterator<List<String>> dataIter;
+    private Configuration conf;
+
+    /**
+     * Called once at initialization.
+     *
+     * @param split the split that defines the range of records to read
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Override
+    public void initialize(InputSplit split) throws IOException, InterruptedException {
+        if (split instanceof ListStringSplit) {
+            ListStringSplit listStringSplit = (ListStringSplit) split;
+            delimitedData = listStringSplit.getData();
+            dataIter = delimitedData.iterator();
+        } else {
+            throw new IllegalArgumentException("Illegal type of input split " + split.getClass().getName());
+        }
+    }
+
+    /**
+     * Called once at initialization.
+     *
+     * @param conf  a configuration for initialization
+     * @param split the split that defines the range of records to read
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        initialize(split);
+    }
+
+    /**
+     * Get the next record
+     *
+     * @return The list of next record
+     */
+    @Override
+    public List<Writable> next() {
+        List<String> next = dataIter.next();
+        invokeListeners(next);
+        List<Writable> ret = new ArrayList<>();
+        for (String s : next)
+            ret.add(new Text(s));
+        return ret;
+    }
+
+    /**
+     * Check whether there are anymore records
+     *
+     * @return Whether there are more records
+     */
+    @Override
+    public boolean hasNext() {
+        return dataIter.hasNext();
+    }
+
+    /**
+     * List of label strings
+     *
+     * @return
+     */
+    @Override
+    public List<String> getLabels() {
+        return null;
+    }
+
+    /**
+     * Reset record reader iterator
+     *
+     */
+    @Override
+    public void reset() {
+        dataIter = delimitedData.iterator();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    /**
+     * Load the record from the given DataInputStream
+     * Unlike {@link #next()} the internal state of the RecordReader is not modified
+     * Implementations of this method should not close the DataInputStream
+     *
+     * @param uri
+     * @param dataInputStream
+     * @throws IOException if error occurs during reading from the input stream
+     */
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        return null;
+    }
+
+    @Override
+    public Record nextRecord() {
+        return new org.datavec.api.records.impl.Record(next(), null);
+    }
+
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        throw new UnsupportedOperationException("Loading from metadata not yet implemented");
+    }
+
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        throw new UnsupportedOperationException("Loading from metadata not yet implemented");
+    }
+
+    /**
+     * Closes this stream and releases any system resources associated
+     * with it. If the stream is already closed then invoking this
+     * method has no effect.
+     * <p>
+     * <p> As noted in {@link AutoCloseable#close()}, cases where the
+     * close may fail require careful attention. It is strongly advised
+     * to relinquish the underlying resources and to internally
+     * <em>mark</em> the {@code Closeable} as closed, prior to throwing
+     * the {@code IOException}.
+     *
+     * @throws IOException if an I/O error occurs
+     */
+    @Override
+    public void close() throws IOException {
+
+    }
+
+    /**
+     * Set the configuration to be used by this object.
+     *
+     * @param conf
+     */
+    @Override
+    public void setConf(Configuration conf) {
+        this.conf = conf;
+    }
+
+    /**
+     * Return the configuration used by this object.
+     */
+    @Override
+    public Configuration getConf() {
+        return conf;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVLineSequenceRecordReader.java

@@ -0,0 +1,107 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.csv;
+
+import org.datavec.api.records.Record;
+import org.datavec.api.records.SequenceRecord;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.reader.SequenceRecordReader;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class CSVLineSequenceRecordReader extends CSVRecordReader implements SequenceRecordReader {
+
+    /**
+     * Default settings: skip 0 lines, use ',' as the delimiter, and '"' for quotes
+     */
+    public CSVLineSequenceRecordReader(){
+        this(0, DEFAULT_DELIMITER, DEFAULT_QUOTE);
+    }
+
+    /**
+     * Skip lines and use delimiter
+     * @param skipNumLines the number of lines to skip
+     * @param delimiter the delimiter
+     */
+    public CSVLineSequenceRecordReader(int skipNumLines, char delimiter) {
+        this(skipNumLines, delimiter, '\"');
+    }
+
+    /**
+     * Skip lines, use delimiter, and strip quotes
+     * @param skipNumLines the number of lines to skip
+     * @param delimiter the delimiter
+     * @param quote the quote to strip
+     */
+    public CSVLineSequenceRecordReader(int skipNumLines, char delimiter, char quote) {
+        super(skipNumLines, delimiter, quote);
+    }
+
+    @Override
+    public List<List<Writable>> sequenceRecord() {
+        return nextSequence().getSequenceRecord();
+    }
+
+    @Override
+    public List<List<Writable>> sequenceRecord(URI uri, DataInputStream dataInputStream) throws IOException {
+        List<Writable> l = record(uri, dataInputStream);
+        List<List<Writable>> out = new ArrayList<>();
+        for(Writable w : l){
+            out.add(Collections.singletonList(w));
+        }
+        return out;
+    }
+
+    @Override
+    public SequenceRecord nextSequence() {
+        return convert(super.nextRecord());
+    }
+
+    @Override
+    public SequenceRecord loadSequenceFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return convert(super.loadFromMetaData(recordMetaData));
+    }
+
+    @Override
+    public List<SequenceRecord> loadSequenceFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        List<Record> toConvert = super.loadFromMetaData(recordMetaDatas);
+        List<SequenceRecord> out = new ArrayList<>();
+        for(Record r  : toConvert){
+            out.add(convert(r));
+        }
+        return out;
+    }
+
+    protected SequenceRecord convert(Record r){
+        List<Writable> line = r.getRecord();
+        List<List<Writable>> out = new ArrayList<>();
+        for(Writable w : line){
+            out.add(Collections.singletonList(w));
+        }
+        return new org.datavec.api.records.impl.SequenceRecord(out, r.getMetaData());
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVMultiSequenceRecordReader.java

@@ -0,0 +1,210 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.csv;
+
+import org.datavec.api.records.SequenceRecord;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataInterval;
+import org.datavec.api.records.reader.SequenceRecordReader;
+import org.datavec.api.writable.Writable;
+import org.nd4j.common.base.Preconditions;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public class CSVMultiSequenceRecordReader extends CSVRecordReader implements SequenceRecordReader {
+
+    public enum Mode {
+        CONCAT,
+        EQUAL_LENGTH,
+        PAD
+    }
+
+    private String sequenceSeparatorRegex;
+    private Mode mode;
+    private Writable padValue;
+
+    /**
+     * Create a sequence reader using the default value for skip lines (0), the default delimiter (',') and the default
+     * quote character ('"').<br>
+     * Note that this constructor cannot be used with {@link Mode#PAD} as the padding value cannot be specified
+     *
+     * @param sequenceSeparatorRegex The sequence separator regex. Use "^$" for "sequences are separated by an empty line
+     * @param mode                   Mode: see {@link CSVMultiSequenceRecordReader} javadoc
+     */
+    public CSVMultiSequenceRecordReader(String sequenceSeparatorRegex, Mode mode){
+        this(0, DEFAULT_DELIMITER, DEFAULT_QUOTE, sequenceSeparatorRegex, mode, null);
+    }
+
+    /**
+     * Create a sequence reader using the default value for skip lines (0), the default delimiter (',') and the default
+     * quote character ('"')
+     *
+     * @param sequenceSeparatorRegex The sequence separator regex. Use "^$" for "sequences are separated by an empty line
+     * @param mode                   Mode: see {@link CSVMultiSequenceRecordReader} javadoc
+     * @param padValue               Padding value for padding short sequences. Only used/allowable with {@link Mode#PAD},
+     *                               should be null otherwise
+     */
+    public CSVMultiSequenceRecordReader(String sequenceSeparatorRegex, Mode mode, Writable padValue){
+        this(0, DEFAULT_DELIMITER, DEFAULT_QUOTE, sequenceSeparatorRegex, mode, padValue);
+    }
+
+    /**
+     * Create a sequence reader using the default value for skip lines (0), the default delimiter (',') and the default
+     * quote character ('"')
+     *
+     * @param skipNumLines           Number of lines to skip
+     * @param elementDelimiter       Delimiter for elements - i.e., ',' if lines are comma separated
+     * @param sequenceSeparatorRegex The sequence separator regex. Use "^$" for "sequences are separated by an empty line
+     * @param mode                   Mode: see {@link CSVMultiSequenceRecordReader} javadoc
+     * @param padValue               Padding value for padding short sequences. Only used/allowable with {@link Mode#PAD},
+     *                               should be null otherwise
+     */
+    public CSVMultiSequenceRecordReader(int skipNumLines, char elementDelimiter, char quote, String sequenceSeparatorRegex,
+                                        Mode mode, Writable padValue){
+        super(skipNumLines, elementDelimiter, quote);
+        Preconditions.checkState(mode != Mode.PAD || padValue != null, "Cannot use Mode.PAD with a null padding value. " +
+                "Padding value must be passed to constructor ");
+        this.sequenceSeparatorRegex = sequenceSeparatorRegex;
+        this.mode = mode;
+        this.padValue = padValue;
+    }
+
+
+    @Override
+    public List<List<Writable>> sequenceRecord() {
+        return nextSequence().getSequenceRecord();
+    }
+
+    @Override
+    public SequenceRecord nextSequence() {
+        if(!hasNext())
+            throw new NoSuchElementException("No next element");
+
+        List<String> lines = new ArrayList<>();
+        int firstLine = lineIndex;
+        int lastLine = lineIndex;
+        while(super.hasNext()){
+            String line = readStringLine();
+            if(line.matches(sequenceSeparatorRegex)){
+                lastLine = lineIndex;
+                break;
+            }
+            lines.add(line);
+        }
+
+        //Process lines
+        URI uri = (locations == null || locations.length < 1 ? null : locations[splitIndex]);
+        List<List<Writable>> out = parseLines(lines, uri, firstLine, lastLine);
+
+
+        return new org.datavec.api.records.impl.SequenceRecord(out, new RecordMetaDataInterval(firstLine, lastLine, uri));
+    }
+
+    private List<List<Writable>> parseLines(List<String> lines, URI uri, int firstLine, int lastLine){
+        List<List<Writable>> out = new ArrayList<>();
+        switch (mode){
+            case CONCAT:
+                //Output is univariate sequence - concat all lines
+                for(String s : lines){
+                    List<Writable> parsed = super.parseLine(s);
+                    for(Writable w : parsed){
+                        out.add(Collections.singletonList(w));
+                    }
+                }
+                break;
+            case EQUAL_LENGTH:
+            case PAD:
+                List<List<Writable>> columnWise = new ArrayList<>();
+                int length = -1;
+                int lineNum = 0;
+                for(String s : lines) {
+                    List<Writable> parsed = super.parseLine(s); //This is one COLUMN
+                    columnWise.add(parsed);
+                    lineNum++;
+                    if(mode == Mode.PAD){
+                        length = Math.max(length, parsed.size());
+                    } else if(length < 0)
+                        length = parsed.size();
+                    else if(mode == Mode.EQUAL_LENGTH){
+                        Preconditions.checkState(parsed.size() == length, "Invalid state: When using CSVMultiSequenceRecordReader, " +
+                                "all lines (columns) must be the same length. Prior columns had " + length + " elements, line " +
+                                lineNum + " in sequence has length " + parsed.size() + " (Sequence position: " + uri +
+                                ", lines " + firstLine + " to " + lastLine + ")");
+                    }
+                }
+
+                if(mode == Mode.PAD){
+                    for(List<Writable> w : columnWise){
+                        while(w.size() < length){
+                            w.add(padValue);
+                        }
+                    }
+                }
+
+                //Transpose: from column-wise to row-wise
+                for( int i=0; i<length; i++ ){
+                    List<Writable> step = new ArrayList<>();
+                    for( int j=0; j<columnWise.size(); j++ ){
+                        step.add(columnWise.get(j).get(i));
+                    }
+                    out.add(step);
+                }
+                break;
+        }
+        return out;
+    }
+
+    @Override
+    public List<List<Writable>> sequenceRecord(URI uri, DataInputStream dataInputStream) throws IOException {
+        List<String> lines = new ArrayList<>();
+        try(BufferedReader br = new BufferedReader(new InputStreamReader(dataInputStream))){
+            String line;
+            while((line = br.readLine()) != null && !line.matches(sequenceSeparatorRegex)){
+                lines.add(line);
+            }
+        }
+
+        return parseLines(lines, uri, 0, lines.size());
+    }
+
+    @Override
+    public SequenceRecord loadSequenceFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        throw new UnsupportedOperationException("Not yet supported");
+    }
+
+    @Override
+    public List<SequenceRecord> loadSequenceFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        throw new UnsupportedOperationException("Not yet supported");
+    }
+
+    @Override
+    public boolean batchesSupported() {
+        return false;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVNLinesSequenceRecordReader.java

@@ -0,0 +1,195 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.csv;
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.SequenceRecord;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataLineInterval;
+import org.datavec.api.records.reader.SequenceRecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+import org.nd4j.common.primitives.Triple;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.*;
+
+public class CSVNLinesSequenceRecordReader extends CSVRecordReader implements SequenceRecordReader {
+
+    public static final String LINES_PER_SEQUENCE = NAME_SPACE + ".nlinespersequence";
+
+    private int nLinesPerSequence;
+    private String delimiter;
+
+    /**
+     * No-arg constructor with the default number of lines per sequence (10)
+     */
+    public CSVNLinesSequenceRecordReader() {
+        this(10);
+    }
+
+    /**
+     * @param nLinesPerSequence    Number of lines in each sequence, use default delemiter(,) between entries in the same line
+     */
+    public CSVNLinesSequenceRecordReader(int nLinesPerSequence) {
+        this(nLinesPerSequence, 0, String.valueOf(CSVRecordReader.DEFAULT_DELIMITER));
+    }
+
+    /**
+     *
+     * @param nLinesPerSequence    Number of lines in each sequences
+     * @param skipNumLines         Number of lines to skip at the start of the file (only skipped once, not per sequence)
+     * @param delimiter            Delimiter between entries in the same line, for example ","
+     */
+    public CSVNLinesSequenceRecordReader(int nLinesPerSequence, int skipNumLines, String delimiter) {
+        super(skipNumLines);
+        this.delimiter = delimiter;
+        this.nLinesPerSequence = nLinesPerSequence;
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        super.initialize(conf, split);
+        this.nLinesPerSequence = conf.getInt(LINES_PER_SEQUENCE, nLinesPerSequence);
+    }
+
+    @Override
+    public List<List<Writable>> sequenceRecord() {
+        if (!super.hasNext()) {
+            throw new NoSuchElementException("No next element");
+        }
+
+        List<List<Writable>> sequence = new ArrayList<>();
+        int count = 0;
+        while (count++ < nLinesPerSequence && super.hasNext()) {
+            sequence.add(super.next());
+        }
+
+        return sequence;
+    }
+
+    @Override
+    public List<List<Writable>> sequenceRecord(URI uri, DataInputStream dataInputStream) throws IOException {
+        throw new UnsupportedOperationException("Reading CSV data from DataInputStream not yet implemented");
+    }
+
+    @Override
+    public SequenceRecord nextSequence() {
+        int lineBefore = lineIndex;
+        List<List<Writable>> record = sequenceRecord();
+        int lineAfter = lineIndex;
+        URI uri = (locations == null || locations.length < 1 ? null : locations[splitIndex]);
+        RecordMetaData meta = new RecordMetaDataLineInterval(lineBefore, lineAfter - 1, uri,
+                        CSVNLinesSequenceRecordReader.class);
+        return new org.datavec.api.records.impl.SequenceRecord(record, meta);
+    }
+
+    @Override
+    public SequenceRecord loadSequenceFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return loadSequenceFromMetaData(Collections.singletonList(recordMetaData)).get(0);
+    }
+
+    @Override
+    public List<SequenceRecord> loadSequenceFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        //First: create a sorted list of the RecordMetaData
+        List<Triple<Integer, RecordMetaDataLineInterval, List<List<Writable>>>> list = new ArrayList<>();
+        Iterator<RecordMetaData> iter = recordMetaDatas.iterator();
+        int count = 0;
+        while (iter.hasNext()) {
+            RecordMetaData rmd = iter.next();
+            if (!(rmd instanceof RecordMetaDataLineInterval)) {
+                throw new IllegalArgumentException(
+                                "Invalid metadata; expected RecordMetaDataLineInterval instance; got: " + rmd);
+            }
+            list.add(new Triple<>(count++, (RecordMetaDataLineInterval) rmd,
+                            (List<List<Writable>>) new ArrayList<List<Writable>>()));
+        }
+
+        //Sort by starting line number:
+        Collections.sort(list, new Comparator<Triple<Integer, RecordMetaDataLineInterval, List<List<Writable>>>>() {
+            @Override
+            public int compare(Triple<Integer, RecordMetaDataLineInterval, List<List<Writable>>> o1,
+                            Triple<Integer, RecordMetaDataLineInterval, List<List<Writable>>> o2) {
+                return Integer.compare(o1.getSecond().getLineNumberStart(), o2.getSecond().getLineNumberStart());
+            }
+        });
+
+        Iterator<String> lineIter = getIterator(0); //TODO handle multi file case...
+        int currentLineIdx = 0;
+        String line = lineIter.next();
+        while (currentLineIdx < skipNumLines) {
+            line = lineIter.next();
+            currentLineIdx++;
+        }
+        for (Triple<Integer, RecordMetaDataLineInterval, List<List<Writable>>> next : list) {
+            int nextStartLine = next.getSecond().getLineNumberStart();
+            int nextEndLine = next.getSecond().getLineNumberEnd();
+            while (currentLineIdx < nextStartLine && lineIter.hasNext()) {
+                line = lineIter.next();
+                currentLineIdx++;
+            }
+            while (currentLineIdx <= nextEndLine && (lineIter.hasNext() || currentLineIdx == nextEndLine)) {
+                String[] split = line.split(this.delimiter, -1);
+                List<Writable> writables = new ArrayList<>();
+                for (String s : split) {
+                    writables.add(new Text(s));
+                }
+                next.getThird().add(writables);
+                currentLineIdx++;
+                if (lineIter.hasNext()) {
+                    line = lineIter.next();
+                }
+            }
+        }
+        closeIfRequired(lineIter);
+
+        //Now, sort by the original order:
+        Collections.sort(list, new Comparator<Triple<Integer, RecordMetaDataLineInterval, List<List<Writable>>>>() {
+            @Override
+            public int compare(Triple<Integer, RecordMetaDataLineInterval, List<List<Writable>>> o1,
+                            Triple<Integer, RecordMetaDataLineInterval, List<List<Writable>>> o2) {
+                return Integer.compare(o1.getFirst(), o2.getFirst());
+            }
+        });
+
+        //And return...
+        List<SequenceRecord> out = new ArrayList<>();
+        for (Triple<Integer, RecordMetaDataLineInterval, List<List<Writable>>> t : list) {
+            out.add(new org.datavec.api.records.impl.SequenceRecord(t.getThird(), t.getSecond()));
+        }
+
+        return out;
+    }
+
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) {
+        throw new UnsupportedOperationException("Not supported");
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVRecordReader.java

@@ -0,0 +1,242 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.csv;
+
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataLine;
+import org.datavec.api.records.reader.impl.LineRecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+import org.nd4j.common.base.Preconditions;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public class CSVRecordReader extends LineRecordReader {
+    private boolean skippedLines = false;
+    protected int skipNumLines = 0;
+    public final static char DEFAULT_DELIMITER = ',';
+    public final static char DEFAULT_QUOTE = '\"';
+    public final static String SKIP_NUM_LINES = NAME_SPACE + ".skipnumlines";
+    public final static String DELIMITER = NAME_SPACE + ".delimiter";
+    public final static String QUOTE = NAME_SPACE + ".quote";
+
+    private SerializableCSVParser csvParser;
+
+    /**
+     * Skip first n lines
+     * @param skipNumLines the number of lines to skip
+     */
+    public CSVRecordReader(int skipNumLines) {
+        this(skipNumLines, DEFAULT_DELIMITER);
+    }
+
+    /**
+     * Create a CSVRecordReader with the specified delimiter
+     * @param delimiter Delimiter character for CSV
+     */
+    public CSVRecordReader(char delimiter){
+        this(0, delimiter);
+    }
+
+    /**
+     * Skip lines and use delimiter
+     * @param skipNumLines the number of lines to skip
+     * @param delimiter the delimiter
+     */
+    public CSVRecordReader(int skipNumLines, char delimiter) {
+        this(skipNumLines, delimiter, '\"');
+    }
+
+    /**
+     *
+     * @param skipNumLines Number of lines to skip
+     * @param delimiter    Delimiter to use
+     * @deprecated This constructor is deprecated; use {@link #CSVRecordReader(int, char)} or
+     * {@link #CSVRecordReader(int, char, char)}
+     */
+    @Deprecated
+    public CSVRecordReader(int skipNumLines, String delimiter){
+        this(skipNumLines, stringDelimToChar(delimiter));
+    }
+
+    private static char stringDelimToChar(String delimiter) {
+        if(delimiter.length() > 1){
+            throw new UnsupportedOperationException("Multi-character delimiters have been deprecated. For quotes, " +
+                    "use CSVRecordReader(int skipNumLines, char delimiter, char quote)");
+        }
+        return delimiter.charAt(0);
+    }
+
+    /**
+     * Skip lines, use delimiter, and strip quotes
+     * @param skipNumLines the number of lines to skip
+     * @param delimiter the delimiter
+     * @param quote the quote to strip
+     */
+    public CSVRecordReader(int skipNumLines, char delimiter, char quote) {
+        this.skipNumLines = skipNumLines;
+        this.csvParser = new SerializableCSVParser(delimiter, quote);
+    }
+
+    /**
+     * Skip lines, use delimiter, and strip quotes
+     * @param skipNumLines the number of lines to skip
+     * @param delimiter the delimiter
+     * @param quote the quote to strip
+     * @deprecated This constructor is deprecated; use {@link #CSVRecordReader(int, char)} or
+     * {@link #CSVRecordReader(int, char, char)}
+     */
+    @Deprecated
+    public CSVRecordReader(int skipNumLines, String delimiter, String quote) {
+        this(skipNumLines, stringDelimToChar(delimiter), stringDelimToChar(quote));
+    }
+
+    public CSVRecordReader() {
+        this(0, DEFAULT_DELIMITER);
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        super.initialize(conf, split);
+        this.skipNumLines = conf.getInt(SKIP_NUM_LINES, this.skipNumLines);
+        this.csvParser = new SerializableCSVParser(conf.getChar(DELIMITER, DEFAULT_DELIMITER), conf.getChar(QUOTE, DEFAULT_QUOTE));
+    }
+
+    private boolean skipLines() {
+        if (!skippedLines && skipNumLines > 0) {
+            for (int i = 0; i < skipNumLines; i++) {
+                if (!super.hasNext()) {
+                    return false;
+                }
+                super.next();
+            }
+            skippedLines = true;
+        }
+        return true;
+    }
+
+    @Override
+    public boolean batchesSupported() {
+        return true;
+    }
+
+    @Override
+    public boolean hasNext() {
+        return skipLines() && super.hasNext();
+    }
+
+    @Override
+    public List<List<Writable>> next(int num) {
+        List<List<Writable>> ret = new ArrayList<>(Math.min(num, 10000));
+        int recordsRead = 0;
+        while(hasNext() && recordsRead++ < num) {
+            ret.add(next());
+        }
+
+        return ret;
+    }
+
+    @Override
+    public List<Writable> next() {
+        if (!skipLines())
+            throw new NoSuchElementException("No next element found!");
+        String val = readStringLine();
+        return parseLine(val);
+    }
+
+    protected List<Writable> parseLine(String line) {
+        String[] split;
+        try {
+            split = csvParser.parseLine(line);
+        } catch(IOException e) {
+            throw new RuntimeException(e);
+        }
+        List<Writable> ret = new ArrayList<>();
+        for (String s : split) {
+            ret.add(new Text(s));
+        }
+        return ret;
+    }
+
+    protected String readStringLine(){
+        Preconditions.checkState(initialized, "RecordReader has not been initialized before use");
+        Text t = (Text) super.next().iterator().next();
+        return t.toString();
+    }
+
+    @Override
+    public Record nextRecord() {
+        List<Writable> next = next();
+        URI uri = (locations == null || locations.length < 1 ? null : locations[splitIndex]);
+        RecordMetaData meta = new RecordMetaDataLine(this.lineIndex - 1, uri, CSVRecordReader.class); //-1 as line number has been incremented already...
+        return new org.datavec.api.records.impl.Record(next, meta);
+    }
+
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return loadFromMetaData(Collections.singletonList(recordMetaData)).get(0);
+    }
+
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        List<Record> list = super.loadFromMetaData(recordMetaDatas);
+
+        for (Record r : list) {
+            String line = r.getRecord().get(0).toString();
+            r.setRecord(parseLine(line));
+        }
+
+        return list;
+    }
+
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        BufferedReader br = new BufferedReader(new InputStreamReader(dataInputStream));
+        for( int i=0; i<skipNumLines; i++ ){
+            br.readLine();
+        }
+        String line = br.readLine();
+        return parseLine(line);
+    }
+
+    @Override
+    public void reset() {
+        super.reset();
+        skippedLines = false;
+    }
+
+    @Override
+    protected void onLocationOpen(URI location) {
+        skippedLines = false;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVRegexRecordReader.java

@@ -0,0 +1,86 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.csv;
+
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class CSVRegexRecordReader extends CSVRecordReader {
+
+    protected String[] regexs = null;
+    protected Pattern[] patterns = null;
+    protected String delimiter;
+    protected String quote;
+
+    /**
+     * Skip lines, use delimiter, strip quotes, and parse each column with a regex
+     * @param skipNumLines the number of lines to skip
+     * @param delimiter the delimiter
+     * @param quote the quote to strip
+     * @param regexs the regexs to parse columns with
+     */
+    public CSVRegexRecordReader(int skipNumLines, String delimiter, String quote, String[] regexs) {
+        super(skipNumLines);
+        this.delimiter = delimiter;
+        this.quote = quote;
+        this.regexs = regexs;
+        if (regexs != null) {
+            patterns = new Pattern[regexs.length];
+            for (int i = 0; i < regexs.length; i++) {
+                if (regexs[i] != null) {
+                    patterns[i] = Pattern.compile(regexs[i]);
+                }
+            }
+        }
+    }
+
+    protected List<Writable> parseLine(String line) {
+        String[] split = line.split(delimiter, -1);
+        List<Writable> ret = new ArrayList<>();
+        for (int i = 0; i < split.length; i++) {
+            String s = split[i];
+            if (quote != null && s.startsWith(quote) && s.endsWith(quote)) {
+                int n = quote.length();
+                s = s.substring(n, s.length() - n).replace(quote + quote, quote);
+            }
+            if (regexs != null && regexs[i] != null) {
+                Matcher m = patterns[i].matcher(s);
+                if (m.matches()) {
+                    for (int j = 1; j <= m.groupCount(); j++) { //Note: Matcher.group(0) is the entire sequence; we only care about groups 1 onward
+                        ret.add(new Text(m.group(j)));
+                    }
+                } else {
+                    throw new IllegalStateException("Invalid line: value does not match regex (regex=\"" + regexs[i]
+                                    + "\"; value=\"" + s + "\"");
+                }
+            } else {
+                ret.add(new Text(s));
+            }
+        }
+        return ret;
+    }
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVSequenceRecordReader.java

@@ -0,0 +1,129 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.csv;
+
+import lombok.SneakyThrows;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.LineIterator;
+import org.datavec.api.records.SequenceRecord;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataURI;
+import org.datavec.api.records.reader.SequenceRecordReader;
+import org.datavec.api.records.reader.impl.FileRecordReader;
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+
+import java.io.*;
+import java.net.URI;
+import java.util.*;
+
+public class CSVSequenceRecordReader extends FileRecordReader implements SequenceRecordReader {
+    private int skipNumLines = 0;
+    private String delimiter = ",";
+
+    public CSVSequenceRecordReader() {
+        this(0, ",");
+    }
+
+    public CSVSequenceRecordReader(int skipNumLines) {
+        this(skipNumLines, ",");
+    }
+
+    public CSVSequenceRecordReader(int skipNumLines, String delimiter) {
+        this.skipNumLines = skipNumLines;
+        this.delimiter = delimiter;
+    }
+
+    @Override
+    public List<List<Writable>> sequenceRecord(URI uri, DataInputStream dataInputStream) throws IOException {
+        invokeListeners(uri);
+        return loadAndClose(dataInputStream);
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public List<List<Writable>> sequenceRecord() {
+        return nextSequence().getSequenceRecord();
+    }
+
+
+    @Override
+    public SequenceRecord nextSequence() {
+        if(!hasNext()){
+            throw new NoSuchElementException("No next element");
+        }
+
+        URI next = locationsIterator.next();
+        invokeListeners(next);
+
+        List<List<Writable>> out = loadAndClose(streamCreatorFn.apply(next));
+        return new org.datavec.api.records.impl.SequenceRecord(out, new RecordMetaDataURI(next));
+    }
+
+    @SneakyThrows
+    private List<List<Writable>> loadAndClose(InputStream inputStream) {
+        LineIterator lineIter = null;
+        try {
+            lineIter = IOUtils.lineIterator(new BufferedReader(new InputStreamReader(inputStream)));
+            return load(lineIter);
+        } finally {
+            if (lineIter != null) {
+                lineIter.close();
+            }
+            IOUtils.closeQuietly(inputStream);
+        }
+    }
+
+    private List<List<Writable>> load(Iterator<String> lineIter) {
+        if (skipNumLines > 0) {
+            int count = 0;
+            while (count++ < skipNumLines && lineIter.hasNext())
+                lineIter.next();
+        }
+
+        List<List<Writable>> out = new ArrayList<>();
+        while (lineIter.hasNext()) {
+            String line = lineIter.next();
+            String[] split = line.split(delimiter);
+            ArrayList<Writable> list = new ArrayList<>();
+            for (String s : split)
+                list.add(new Text(s));
+            out.add(list);
+        }
+        return out;
+    }
+
+    @Override
+    public SequenceRecord loadSequenceFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return loadSequenceFromMetaData(Collections.singletonList(recordMetaData)).get(0);
+    }
+
+    @Override
+    public List<SequenceRecord> loadSequenceFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        List<SequenceRecord> out = new ArrayList<>();
+        for (RecordMetaData meta : recordMetaDatas) {
+            File next = new File(meta.getURI());
+            List<List<Writable>> sequence = loadAndClose(new FileInputStream(next));
+            out.add(new org.datavec.api.records.impl.SequenceRecord(sequence, meta));
+        }
+        return out;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVVariableSlidingWindowRecordReader.java

@@ -0,0 +1,205 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.csv;
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.SequenceRecord;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataLineInterval;
+import org.datavec.api.records.reader.SequenceRecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.*;
+
+public class CSVVariableSlidingWindowRecordReader extends CSVRecordReader implements SequenceRecordReader {
+
+    public static final String LINES_PER_SEQUENCE = NAME_SPACE + ".nlinespersequence";
+
+    private int maxLinesPerSequence;
+    private int stride;
+    private LinkedList<List<Writable>> queue;
+    private boolean exhausted;
+
+    /**
+     * No-arg constructor with the default number of lines per sequence (10)
+     */
+    public CSVVariableSlidingWindowRecordReader() {
+        this(10, 1);
+    }
+
+    /**
+     * @param maxLinesPerSequence Number of lines in each sequence, use default delemiter(,) between entries in the same line
+     */
+    public CSVVariableSlidingWindowRecordReader(int maxLinesPerSequence) {
+        this(maxLinesPerSequence, 0, 1, CSVRecordReader.DEFAULT_DELIMITER);
+    }
+
+    /**
+     * @param maxLinesPerSequence Number of lines in each sequence, use default delemiter(,) between entries in the same line
+     * @param stride Number of lines between records (increment window > 1 line)
+     */
+    public CSVVariableSlidingWindowRecordReader(int maxLinesPerSequence, int stride) {
+        this(maxLinesPerSequence, 0, stride, CSVRecordReader.DEFAULT_DELIMITER);
+    }
+
+    /**
+     * @param maxLinesPerSequence Number of lines in each sequence, use default delemiter(,) between entries in the same line
+     * @param stride Number of lines between records (increment window > 1 line)
+     * @deprecated Use the constructor using char for delimiter instead
+     */
+    @Deprecated
+    public CSVVariableSlidingWindowRecordReader(int maxLinesPerSequence, int stride, String delimiter) {
+        this(maxLinesPerSequence, 0, stride, CSVRecordReader.DEFAULT_DELIMITER);
+    }
+
+    /**
+     *
+     * @param maxLinesPerSequence Number of lines in each sequences
+     * @param skipNumLines Number of lines to skip at the start of the file (only skipped once, not per sequence)
+     * @param stride Number of lines between records (increment window > 1 line)
+     * @param delimiter Delimiter between entries in the same line, for example ","
+     * @deprecated Use the constructor using char for delimiter instead
+     */
+    @Deprecated
+    public CSVVariableSlidingWindowRecordReader(int maxLinesPerSequence, int skipNumLines, int stride, String delimiter) {
+        super(skipNumLines, delimiter.charAt(0));
+        if(stride < 1)
+            throw new IllegalArgumentException("Stride must be greater than 1");
+
+        this.maxLinesPerSequence = maxLinesPerSequence;
+        this.stride = stride;
+        this.queue = new LinkedList<>();
+        this.exhausted = false;
+    }
+
+    /**
+     * @param maxLinesPerSequence Number of lines in each sequence, use default delemiter(,) between entries in the same line
+     * @param stride Number of lines between records (increment window > 1 line)
+     */
+    public CSVVariableSlidingWindowRecordReader(int maxLinesPerSequence, int stride, char delimiter) {
+        this(maxLinesPerSequence, 0, stride, delimiter);
+    }
+
+    /**
+     *
+     * @param maxLinesPerSequence Number of lines in each sequences
+     * @param skipNumLines Number of lines to skip at the start of the file (only skipped once, not per sequence)
+     * @param stride Number of lines between records (increment window > 1 line)
+     * @param delimiter Delimiter between entries in the same line, for example ","
+     */
+    public CSVVariableSlidingWindowRecordReader(int maxLinesPerSequence, int skipNumLines, int stride, char delimiter) {
+        super(skipNumLines, delimiter);
+        if(stride < 1)
+            throw new IllegalArgumentException("Stride must be greater than 1");
+
+        this.maxLinesPerSequence = maxLinesPerSequence;
+        this.stride = stride;
+        this.queue = new LinkedList<>();
+        this.exhausted = false;
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        super.initialize(conf, split);
+        this.maxLinesPerSequence = conf.getInt(LINES_PER_SEQUENCE, maxLinesPerSequence);
+    }
+
+    @Override
+    public boolean hasNext() {
+        boolean moreInCsv = super.hasNext();
+        boolean moreInQueue = !queue.isEmpty();
+        return moreInCsv || moreInQueue;
+    }
+
+    @Override
+    public List<List<Writable>> sequenceRecord() {
+        // try polling next(), otherwise empty the queue
+        // loop according to stride size
+        for(int i = 0; i < stride; i++) {
+            if(super.hasNext())
+                queue.addFirst(super.next());
+            else
+                exhausted = true;
+
+            if (exhausted && queue.size() < 1)
+                throw new NoSuchElementException("No next element");
+
+            if (queue.size() > maxLinesPerSequence || exhausted)
+                queue.pollLast();
+        }
+
+        List<List<Writable>> sequence = new ArrayList<>();
+        sequence.addAll(queue);
+
+        if(exhausted && queue.size()==1)
+            queue.pollLast();
+
+        return sequence;
+    }
+
+    @Override
+    public List<List<Writable>> sequenceRecord(URI uri, DataInputStream dataInputStream) throws IOException {
+        throw new UnsupportedOperationException("Reading CSV data from DataInputStream not yet implemented");
+    }
+
+    @Override
+    public SequenceRecord nextSequence() {
+        int lineBefore = lineIndex;
+        List<List<Writable>> record = sequenceRecord();
+        int lineAfter = lineIndex + queue.size();
+        URI uri = (locations == null || locations.length < 1 ? null : locations[splitIndex]);
+        RecordMetaData meta = new RecordMetaDataLineInterval(lineBefore, lineAfter - 1, uri,
+                        CSVVariableSlidingWindowRecordReader.class);
+        return new org.datavec.api.records.impl.SequenceRecord(record, meta);
+    }
+
+    @Override
+    public SequenceRecord loadSequenceFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return loadSequenceFromMetaData(Collections.singletonList(recordMetaData)).get(0);
+    }
+
+    @Override
+    public List<SequenceRecord> loadSequenceFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public void reset() {
+        super.reset();
+        queue = new LinkedList<>();
+        exhausted = false;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/SerializableCSVParser.java

@@ -0,0 +1,321 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.csv;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+public class SerializableCSVParser implements Serializable {
+
+    private final char separator;
+
+    private final char quotechar;
+
+    private final char escape;
+
+    private final boolean strictQuotes;
+
+    private String pending;
+    private boolean inField = false;
+
+    private final boolean ignoreLeadingWhiteSpace;
+
+    /**
+     * The default separator to use if none is supplied to the constructor.
+     */
+    public static final char DEFAULT_SEPARATOR = ',';
+
+    public static final int INITIAL_READ_SIZE = 128;
+
+    /**
+     * The default quote character to use if none is supplied to the
+     * constructor.
+     */
+    public static final char DEFAULT_QUOTE_CHARACTER = '"';
+
+
+    /**
+     * The default escape character to use if none is supplied to the
+     * constructor.
+     */
+    public static final char DEFAULT_ESCAPE_CHARACTER = '\\';
+
+    /**
+     * The default strict quote behavior to use if none is supplied to the
+     * constructor
+     */
+    public static final boolean DEFAULT_STRICT_QUOTES = false;
+
+    /**
+     * The default leading whitespace behavior to use if none is supplied to the
+     * constructor
+     */
+    public static final boolean DEFAULT_IGNORE_LEADING_WHITESPACE = true;
+
+    /**
+     * This is the "null" character - if a value is set to this then it is ignored.
+     * I.E. if the quote character is set to null then there is no quote character.
+     */
+    public static final char NULL_CHARACTER = '\0';
+
+    /**
+     * Constructs CSVParser using a comma for the separator.
+     */
+    public SerializableCSVParser() {
+        this(DEFAULT_SEPARATOR, DEFAULT_QUOTE_CHARACTER, DEFAULT_ESCAPE_CHARACTER);
+    }
+
+    /**
+     * Constructs CSVParser with supplied separator.
+     *
+     * @param separator the delimiter to use for separating entries.
+     */
+    public SerializableCSVParser(char separator) {
+        this(separator, DEFAULT_QUOTE_CHARACTER, DEFAULT_ESCAPE_CHARACTER);
+    }
+
+
+    /**
+     * Constructs CSVParser with supplied separator and quote char.
+     *
+     * @param separator the delimiter to use for separating entries
+     * @param quotechar the character to use for quoted elements
+     */
+    public SerializableCSVParser(char separator, char quotechar) {
+        this(separator, quotechar, DEFAULT_ESCAPE_CHARACTER);
+    }
+
+    /**
+     * Constructs CSVReader with supplied separator and quote char.
+     *
+     * @param separator the delimiter to use for separating entries
+     * @param quotechar the character to use for quoted elements
+     * @param escape    the character to use for escaping a separator or quote
+     */
+    public SerializableCSVParser(char separator, char quotechar, char escape) {
+        this(separator, quotechar, escape, DEFAULT_STRICT_QUOTES);
+    }
+
+    /**
+     * Constructs CSVReader with supplied separator and quote char.
+     * Allows setting the "strict quotes" flag
+     *
+     * @param separator    the delimiter to use for separating entries
+     * @param quotechar    the character to use for quoted elements
+     * @param escape       the character to use for escaping a separator or quote
+     * @param strictQuotes if true, characters outside the quotes are ignored
+     */
+    public SerializableCSVParser(char separator, char quotechar, char escape, boolean strictQuotes) {
+        this(separator, quotechar, escape, strictQuotes, DEFAULT_IGNORE_LEADING_WHITESPACE);
+    }
+
+    /**
+     * Constructs CSVReader with supplied separator and quote char.
+     * Allows setting the "strict quotes" and "ignore leading whitespace" flags
+     *
+     * @param separator               the delimiter to use for separating entries
+     * @param quotechar               the character to use for quoted elements
+     * @param escape                  the character to use for escaping a separator or quote
+     * @param strictQuotes            if true, characters outside the quotes are ignored
+     * @param ignoreLeadingWhiteSpace if true, white space in front of a quote in a field is ignored
+     */
+    public SerializableCSVParser(char separator, char quotechar, char escape, boolean strictQuotes, boolean ignoreLeadingWhiteSpace) {
+        if (anyCharactersAreTheSame(separator, quotechar, escape)) {
+            throw new UnsupportedOperationException("The separator, quote, and escape characters must be different!");
+        }
+        if (separator == NULL_CHARACTER) {
+            throw new UnsupportedOperationException("The separator character must be defined!");
+        }
+        this.separator = separator;
+        this.quotechar = quotechar;
+        this.escape = escape;
+        this.strictQuotes = strictQuotes;
+        this.ignoreLeadingWhiteSpace = ignoreLeadingWhiteSpace;
+    }
+
+    private boolean anyCharactersAreTheSame(char separator, char quotechar, char escape) {
+        return isSameCharacter(separator, quotechar) || isSameCharacter(separator, escape) || isSameCharacter(quotechar, escape);
+    }
+
+    private boolean isSameCharacter(char c1, char c2) {
+        return c1 != NULL_CHARACTER && c1 == c2;
+    }
+
+    /**
+     * @return true if something was left over from last call(s)
+     */
+    public boolean isPending() {
+        return pending != null;
+    }
+
+    public String[] parseLineMulti(String nextLine) throws IOException {
+        return parseLine(nextLine, true);
+    }
+
+    public String[] parseLine(String nextLine) throws IOException {
+        return parseLine(nextLine, false);
+    }
+
+    /**
+     * Parses an incoming String and returns an array of elements.
+     *
+     * @param nextLine the string to parse
+     * @param multi
+     * @return the comma-tokenized list of elements, or null if nextLine is null
+     * @throws IOException if bad things happen during the read
+     */
+    private String[] parseLine(String nextLine, boolean multi) throws IOException {
+
+        if (!multi && pending != null) {
+            pending = null;
+        }
+
+        if (nextLine == null) {
+            if (pending != null) {
+                String s = pending;
+                pending = null;
+                return new String[]{s};
+            } else {
+                return null;
+            }
+        }
+
+        List<String> tokensOnThisLine = new ArrayList<String>();
+        StringBuilder sb = new StringBuilder(INITIAL_READ_SIZE);
+        boolean inQuotes = false;
+        if (pending != null) {
+            sb.append(pending);
+            pending = null;
+            inQuotes = true;
+        }
+        for (int i = 0; i < nextLine.length(); i++) {
+
+            char c = nextLine.charAt(i);
+            if (c == this.escape) {
+                if (isNextCharacterEscapable(nextLine, inQuotes || inField, i)) {
+                    sb.append(nextLine.charAt(i + 1));
+                    i++;
+                }
+            } else if (c == quotechar) {
+                if (isNextCharacterEscapedQuote(nextLine, inQuotes || inField, i)) {
+                    sb.append(nextLine.charAt(i + 1));
+                    i++;
+                } else {
+                    //inQuotes = !inQuotes;
+
+                    // the tricky case of an embedded quote in the middle: a,bc"d"ef,g
+                    if (!strictQuotes) {
+                        if (i > 2 //not on the beginning of the line
+                                && nextLine.charAt(i - 1) != this.separator //not at the beginning of an escape sequence
+                                && nextLine.length() > (i + 1) &&
+                                nextLine.charAt(i + 1) != this.separator //not at the	end of an escape sequence
+                                ) {
+
+                            if (ignoreLeadingWhiteSpace && sb.length() > 0 && isAllWhiteSpace(sb)) {
+                                sb.setLength(0);  //discard white space leading up to quote
+                            } else {
+                                sb.append(c);
+                                //continue;
+                            }
+
+                        }
+                    }
+
+                    inQuotes = !inQuotes;
+                }
+                inField = !inField;
+            } else if (c == separator && !inQuotes) {
+                tokensOnThisLine.add(sb.toString());
+                sb.setLength(0); // start work on next token
+                inField = false;
+            } else {
+                if (!strictQuotes || inQuotes) {
+                    sb.append(c);
+                    inField = true;
+                }
+            }
+        }
+        // line is done - check status
+        if (inQuotes) {
+            if (multi) {
+                // continuing a quoted section, re-append newline
+                sb.append("\n");
+                pending = sb.toString();
+                sb = null; // this partial content is not to be added to field list yet
+            } else {
+                throw new IOException("Un-terminated quoted field at end of CSV line");
+            }
+        }
+        if (sb != null) {
+            tokensOnThisLine.add(sb.toString());
+        }
+        return tokensOnThisLine.toArray(new String[tokensOnThisLine.size()]);
+
+    }
+
+    /**
+     * precondition: the current character is a quote or an escape
+     *
+     * @param nextLine the current line
+     * @param inQuotes true if the current context is quoted
+     * @param i        current index in line
+     * @return true if the following character is a quote
+     */
+    private boolean isNextCharacterEscapedQuote(String nextLine, boolean inQuotes, int i) {
+        return inQuotes  // we are in quotes, therefore there can be escaped quotes in here.
+                && nextLine.length() > (i + 1)  // there is indeed another character to check.
+                && nextLine.charAt(i + 1) == quotechar;
+    }
+
+    /**
+     * precondition: the current character is an escape
+     *
+     * @param nextLine the current line
+     * @param inQuotes true if the current context is quoted
+     * @param i        current index in line
+     * @return true if the following character is a quote
+     */
+    protected boolean isNextCharacterEscapable(String nextLine, boolean inQuotes, int i) {
+        return inQuotes  // we are in quotes, therefore there can be escaped quotes in here.
+                && nextLine.length() > (i + 1)  // there is indeed another character to check.
+                && (nextLine.charAt(i + 1) == quotechar || nextLine.charAt(i + 1) == this.escape);
+    }
+
+    /**
+     * precondition: sb.length() > 0
+     *
+     * @param sb A sequence of characters to examine
+     * @return true if every character in the sequence is whitespace
+     */
+    protected boolean isAllWhiteSpace(CharSequence sb) {
+        boolean result = true;
+        for (int i = 0; i < sb.length(); i++) {
+            char c = sb.charAt(i);
+
+            if (!Character.isWhitespace(c)) {
+                return false;
+            }
+        }
+        return result;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchRecordReader.java

@@ -0,0 +1,168 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.filebatch;
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.listener.RecordListener;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Writable;
+import org.nd4j.common.loader.FileBatch;
+import org.nd4j.common.base.Preconditions;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class FileBatchRecordReader implements RecordReader {
+
+    private final RecordReader recordReader;
+    private final FileBatch fileBatch;
+    private int position = 0;
+
+    /**
+     * @param rr        Underlying record reader to read files from
+     * @param fileBatch File batch to read files from
+     */
+    public FileBatchRecordReader(RecordReader rr, FileBatch fileBatch){
+        this.recordReader = rr;
+        this.fileBatch = fileBatch;
+    }
+
+
+    @Override
+    public void initialize(InputSplit split) throws IOException, InterruptedException {
+        //No op
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        //No op
+    }
+
+    @Override
+    public boolean batchesSupported() {
+        return false;
+    }
+
+    @Override
+    public List<List<Writable>> next(int num) {
+        List<List<Writable>> out = new ArrayList<>(Math.min(num, 10000));
+        for( int i=0; i<num && hasNext(); i++ ){
+            out.add(next());
+        }
+        return out;
+    }
+
+    @Override
+    public List<Writable> next() {
+        Preconditions.checkState(hasNext(), "No next element");
+
+        byte[] fileBytes = fileBatch.getFileBytes().get(position);
+        String origPath = fileBatch.getOriginalUris().get(position);
+
+        List<Writable> out;
+        try {
+            out = recordReader.record(URI.create(origPath), new DataInputStream(new ByteArrayInputStream(fileBytes)));
+        } catch (IOException e){
+            throw new RuntimeException("Error reading from file bytes");
+        }
+
+        position++;
+        return out;
+    }
+
+    @Override
+    public boolean hasNext() {
+        return position < fileBatch.getFileBytes().size();
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return recordReader.getLabels();
+    }
+
+    @Override
+    public void reset() {
+        position = 0;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public Record nextRecord() {
+        return new org.datavec.api.records.impl.Record(next(), null);
+    }
+
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return recordReader.loadFromMetaData(recordMetaData);
+    }
+
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        return recordReader.loadFromMetaData(recordMetaDatas);
+    }
+
+    @Override
+    public List<RecordListener> getListeners() {
+        return null;
+    }
+
+    @Override
+    public void setListeners(RecordListener... listeners) {
+        recordReader.setListeners(listeners);
+    }
+
+    @Override
+    public void setListeners(Collection<RecordListener> listeners) {
+        recordReader.setListeners(listeners);
+    }
+
+    @Override
+    public void close() throws IOException {
+        recordReader.close();
+    }
+
+    @Override
+    public void setConf(Configuration conf) {
+        recordReader.setConf(conf);
+    }
+
+    @Override
+    public Configuration getConf() {
+        return recordReader.getConf();
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchSequenceRecordReader.java

@@ -0,0 +1,188 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.filebatch;
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.SequenceRecord;
+import org.datavec.api.records.listener.RecordListener;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.reader.SequenceRecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Writable;
+import org.nd4j.common.loader.FileBatch;
+import org.nd4j.common.base.Preconditions;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Collection;
+import java.util.List;
+
+public class FileBatchSequenceRecordReader implements SequenceRecordReader {
+
+    private final SequenceRecordReader recordReader;
+    private final FileBatch fileBatch;
+    private int position = 0;
+
+    /**
+     * @param seqRR     Underlying record reader to read files from
+     * @param fileBatch File batch to read files from
+     */
+    public FileBatchSequenceRecordReader(SequenceRecordReader seqRR, FileBatch fileBatch){
+        this.recordReader = seqRR;
+        this.fileBatch = fileBatch;
+    }
+
+    @Override
+    public List<List<Writable>> sequenceRecord() {
+        Preconditions.checkState(hasNext(), "No next element");
+
+        byte[] fileBytes = fileBatch.getFileBytes().get(position);
+        String origPath = fileBatch.getOriginalUris().get(position);
+
+        List<List<Writable>> out;
+        try {
+            out = recordReader.sequenceRecord(URI.create(origPath), new DataInputStream(new ByteArrayInputStream(fileBytes)));
+        } catch (IOException e){
+            throw new RuntimeException("Error reading from file bytes");
+        }
+
+        position++;
+        return out;
+    }
+
+    @Override
+    public List<List<Writable>> sequenceRecord(URI uri, DataInputStream dataInputStream) throws IOException {
+        return recordReader.sequenceRecord(uri, dataInputStream);
+    }
+
+    @Override
+    public SequenceRecord nextSequence() {
+        return new org.datavec.api.records.impl.SequenceRecord(sequenceRecord(), null);
+    }
+
+    @Override
+    public SequenceRecord loadSequenceFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return recordReader.loadSequenceFromMetaData(recordMetaData);
+    }
+
+    @Override
+    public List<SequenceRecord> loadSequenceFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        return recordReader.loadSequenceFromMetaData(recordMetaDatas);
+    }
+
+    @Override
+    public void initialize(InputSplit split) throws IOException, InterruptedException {
+        //No op
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        //No op
+    }
+
+    @Override
+    public boolean batchesSupported() {
+        return false;
+    }
+
+    @Override
+    public List<List<Writable>> next(int num) {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public List<Writable> next() {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public boolean hasNext() {
+        return position < fileBatch.getFileBytes().size();
+    }
+
+    @Override
+    public List<String> getLabels() {
+        return recordReader.getLabels();
+    }
+
+    @Override
+    public void reset() {
+        position = 0;
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public Record nextRecord() {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public List<RecordListener> getListeners() {
+        return recordReader.getListeners();
+    }
+
+    @Override
+    public void setListeners(RecordListener... listeners) {
+        recordReader.setListeners(listeners);
+    }
+
+    @Override
+    public void setListeners(Collection<RecordListener> listeners) {
+        recordReader.setListeners(listeners);
+    }
+
+    @Override
+    public void close() throws IOException {
+        recordReader.close();
+    }
+
+    @Override
+    public void setConf(Configuration conf) {
+        recordReader.setConf(conf);
+    }
+
+    @Override
+    public Configuration getConf() {
+        return recordReader.getConf();
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/inmemory/InMemoryRecordReader.java

@@ -0,0 +1,244 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.inmemory;
+
+import lombok.Data;
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.listener.RecordListener;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.*;
+
+@Data
+public class InMemoryRecordReader implements RecordReader {
+    private List<List<Writable>> records;
+    private Iterator<List<Writable>> iter;
+    private List<String> labels;
+    private List<RecordListener> recordListeners;
+    private Configuration configuration;
+
+    public InMemoryRecordReader(List<List<Writable>> records) {
+        this.records = records;
+        this.iter = records.iterator();
+    }
+
+    /**
+     * Called once at initialization.
+     *
+     * @param split the split that defines the range of records to read
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Override
+    public void initialize(InputSplit split) throws IOException, InterruptedException {
+
+    }
+
+    /**
+     * Called once at initialization.
+     *
+     * @param conf  a configuration for initialization
+     * @param split the split that defines the range of records to read
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+
+    }
+
+    @Override
+    public boolean batchesSupported() {
+        return false;
+    }
+
+    @Override
+    public List<List<Writable>> next(int num) {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Get the next record
+     *
+     * @return
+     */
+    @Override
+    public List<Writable> next() {
+        return iter.next();
+    }
+
+    /**
+     * Whether there are anymore records
+     *
+     * @return
+     */
+    @Override
+    public boolean hasNext() {
+        return iter.hasNext();
+    }
+
+    /**
+     * List of label strings
+     *
+     * @return
+     */
+    @Override
+    public List<String> getLabels() {
+        return labels;
+    }
+
+    /**
+     * Reset record reader iterator
+     *
+     * @return
+     */
+    @Override
+    public void reset() {
+        iter = records.iterator();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    /**
+     * Load the record from the given DataInputStream
+     * Unlike {@link #next()} the internal state of the RecordReader is not modified
+     * Implementations of this method should not close the DataInputStream
+     *
+     * @param uri
+     * @param dataInputStream
+     * @throws IOException if error occurs during reading from the input stream
+     */
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Similar to {@link #next()}, but returns a {@link Record} object, that may include metadata such as the source
+     * of the data
+     *
+     * @return next record
+     */
+    @Override
+    public Record nextRecord() {
+        return new org.datavec.api.records.impl.Record(iter.next(), null);
+    }
+
+    /**
+     * Load a single record from the given {@link RecordMetaData} instance<br>
+     * Note: that for data that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to
+     * load multiple records at once using {@link #loadFromMetaData(List)}
+     *
+     * @param recordMetaData Metadata for the record that we want to load from
+     * @return Single record for the given RecordMetaData instance
+     * @throws IOException If I/O error occurs during loading
+     */
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Load multiple records from the given a list of {@link RecordMetaData} instances<br>
+     *
+     * @param recordMetaDatas Metadata for the records that we want to load from
+     * @return Multiple records for the given RecordMetaData instances
+     * @throws IOException If I/O error occurs during loading
+     */
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Get the record listeners for this record reader.
+     */
+    @Override
+    public List<RecordListener> getListeners() {
+        return recordListeners;
+    }
+
+    /**
+     * Set the record listeners for this record reader.
+     *
+     * @param listeners
+     */
+    @Override
+    public void setListeners(RecordListener... listeners) {
+        this.recordListeners = Arrays.asList(listeners);
+    }
+
+    /**
+     * Set the record listeners for this record reader.
+     *
+     * @param listeners
+     */
+    @Override
+    public void setListeners(Collection<RecordListener> listeners) {
+        this.recordListeners = new ArrayList<>(listeners);
+    }
+
+    /**
+     * Closes this stream and releases any system resources associated
+     * with it. If the stream is already closed then invoking this
+     * method has no effect.
+     * <p>
+     * <p> As noted in {@link AutoCloseable#close()}, cases where the
+     * close may fail require careful attention. It is strongly advised
+     * to relinquish the underlying resources and to internally
+     * <em>mark</em> the {@code Closeable} as closed, prior to throwing
+     * the {@code IOException}.
+     *
+     * @throws IOException if an I/O error occurs
+     */
+    @Override
+    public void close() throws IOException {
+
+    }
+
+    /**
+     * Set the configuration to be used by this object.
+     *
+     * @param conf
+     */
+    @Override
+    public void setConf(Configuration conf) {
+        this.configuration = conf;
+    }
+
+    /**
+     * Return the configuration used by this object.
+     */
+    @Override
+    public Configuration getConf() {
+        return configuration;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/inmemory/InMemorySequenceRecordReader.java

@@ -0,0 +1,307 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.inmemory;
+
+import lombok.Data;
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.SequenceRecord;
+import org.datavec.api.records.listener.RecordListener;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.reader.SequenceRecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.*;
+
+@Data
+public class InMemorySequenceRecordReader implements SequenceRecordReader {
+    private List<List<List<Writable>>> records;
+    private Iterator<List<List<Writable>>> iter;
+    private List<String> labels;
+    private List<RecordListener> recordListeners;
+    private Configuration configuration;
+
+    public InMemorySequenceRecordReader(List<List<List<Writable>>> records) {
+        this.records = records;
+        this.iter = records.iterator();
+
+    }
+
+    /**
+     * Set the configuration to be used by this object.
+     *
+     * @param conf
+     */
+    @Override
+    public void setConf(Configuration conf) {
+        this.configuration = conf;
+    }
+
+    /**
+     * Return the configuration used by this object.
+     */
+    @Override
+    public Configuration getConf() {
+        return configuration;
+    }
+
+    /**
+     * Returns a sequence record.
+     *
+     * @return a sequence of records
+     */
+    @Override
+    public List<List<Writable>> sequenceRecord() {
+        return iter.next();
+    }
+
+    @Override
+    public boolean batchesSupported() {
+        return false;
+    }
+
+    @Override
+    public List<List<Writable>> next(int num) {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Load a sequence record from the given DataInputStream
+     * Unlike {@link #next()} the internal state of the RecordReader is not modified
+     * Implementations of this method should not close the DataInputStream
+     *
+     * @param uri
+     * @param dataInputStream
+     * @throws IOException if error occurs during reading from the input stream
+     */
+    @Override
+    public List<List<Writable>> sequenceRecord(URI uri, DataInputStream dataInputStream) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Similar to {@link #sequenceRecord()}, but returns a {@link Record} object, that may include metadata such as the source
+     * of the data
+     *
+     * @return next sequence record
+     */
+    @Override
+    public SequenceRecord nextSequence() {
+        return new org.datavec.api.records.impl.SequenceRecord(sequenceRecord(), null);
+    }
+
+    /**
+     * Load a single sequence record from the given {@link RecordMetaData} instance<br>
+     * Note: that for data that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to
+     * load multiple records at once using {@link #loadSequenceFromMetaData(List)}
+     *
+     * @param recordMetaData Metadata for the sequence record that we want to load from
+     * @return Single sequence record for the given RecordMetaData instance
+     * @throws IOException If I/O error occurs during loading
+     */
+    @Override
+    public SequenceRecord loadSequenceFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Load multiple sequence records from the given a list of {@link RecordMetaData} instances<br>
+     *
+     * @param recordMetaDatas Metadata for the records that we want to load from
+     * @return Multiple sequence record for the given RecordMetaData instances
+     * @throws IOException If I/O error occurs during loading
+     */
+    @Override
+    public List<SequenceRecord> loadSequenceFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Called once at initialization.
+     *
+     * @param split the split that defines the range of records to read
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Override
+    public void initialize(InputSplit split) throws IOException, InterruptedException {
+
+    }
+
+    /**
+     * Called once at initialization.
+     *
+     * @param conf  a configuration for initialization
+     * @param split the split that defines the range of records to read
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+
+    }
+
+    /**
+     * Get the next record
+     *
+     * @return
+     */
+    @Override
+    public List<Writable> next() {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Whether there are anymore records
+     *
+     * @return
+     */
+    @Override
+    public boolean hasNext() {
+        return iter.hasNext();
+    }
+
+    /**
+     * List of label strings
+     *
+     * @return
+     */
+    @Override
+    public List<String> getLabels() {
+        return labels;
+    }
+
+    /**
+     * Reset record reader iterator
+     *
+     * @return
+     */
+    @Override
+    public void reset() {
+        this.iter = records.iterator();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    /**
+     * Load the record from the given DataInputStream
+     * Unlike {@link #next()} the internal state of the RecordReader is not modified
+     * Implementations of this method should not close the DataInputStream
+     *
+     * @param uri
+     * @param dataInputStream
+     * @throws IOException if error occurs during reading from the input stream
+     */
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        return null;
+    }
+
+    /**
+     * Similar to {@link #next()}, but returns a {@link Record} object, that may include metadata such as the source
+     * of the data
+     *
+     * @return next record
+     */
+    @Override
+    public Record nextRecord() {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Load a single record from the given {@link RecordMetaData} instance<br>
+     * Note: that for data that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to
+     * load multiple records at once using {@link #loadFromMetaData(List)}
+     *
+     * @param recordMetaData Metadata for the record that we want to load from
+     * @return Single record for the given RecordMetaData instance
+     * @throws IOException If I/O error occurs during loading
+     */
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Load multiple records from the given a list of {@link RecordMetaData} instances<br>
+     *
+     * @param recordMetaDatas Metadata for the records that we want to load from
+     * @return Multiple records for the given RecordMetaData instances
+     * @throws IOException If I/O error occurs during loading
+     */
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Get the record listeners for this record reader.
+     */
+    @Override
+    public List<RecordListener> getListeners() {
+        return recordListeners;
+    }
+
+    /**
+     * Set the record listeners for this record reader.
+     *
+     * @param listeners
+     */
+    @Override
+    public void setListeners(RecordListener... listeners) {
+        this.recordListeners = Arrays.asList(listeners);
+    }
+
+    /**
+     * Set the record listeners for this record reader.
+     *
+     * @param listeners
+     */
+    @Override
+    public void setListeners(Collection<RecordListener> listeners) {
+        this.recordListeners = new ArrayList<>(listeners);
+    }
+
+    /**
+     * Closes this stream and releases any system resources associated
+     * with it. If the stream is already closed then invoking this
+     * method has no effect.
+     * <p>
+     * <p> As noted in {@link AutoCloseable#close()}, cases where the
+     * close may fail require careful attention. It is strongly advised
+     * to relinquish the underlying resources and to internally
+     * <em>mark</em> the {@code Closeable} as closed, prior to throwing
+     * the {@code IOException}.
+     *
+     * @throws IOException if an I/O error occurs
+     */
+    @Override
+    public void close() throws IOException {
+
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/FieldSelection.java

@@ -0,0 +1,80 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.jackson;
+
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+public class FieldSelection implements Serializable {
+
+    public static final Writable DEFAULT_MISSING_VALUE = new Text("");
+
+    private List<String[]> fieldPaths;
+    private List<Writable> valueIfMissing;
+
+    private FieldSelection(Builder builder) {
+        this.fieldPaths = builder.fieldPaths;
+        this.valueIfMissing = builder.valueIfMissing;
+    }
+
+    public List<String[]> getFieldPaths() {
+        return fieldPaths;
+    }
+
+    public List<Writable> getValueIfMissing() {
+        return valueIfMissing;
+    }
+
+    public int getNumFields() {
+        return fieldPaths.size();
+    }
+
+    public static class Builder {
+
+        private List<String[]> fieldPaths = new ArrayList<>();
+        private List<Writable> valueIfMissing = new ArrayList<>();
+
+
+        /**
+         *
+         * @param fieldPath    Path to the field. For example, {"a", "b", "c"} would be a field named c, in an object b,
+         *                     where b is in an object a
+         */
+        public Builder addField(String... fieldPath) {
+            return addField(DEFAULT_MISSING_VALUE, fieldPath);
+        }
+
+        public Builder addField(Writable valueIfMissing, String... fieldPath) {
+            fieldPaths.add(fieldPath);
+            this.valueIfMissing.add(valueIfMissing);
+            return this;
+        }
+
+        public FieldSelection build() {
+            return new FieldSelection(this);
+        }
+    }
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonLineRecordReader.java

@@ -0,0 +1,55 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.jackson;
+
+import java.util.List;
+
+import org.datavec.api.records.reader.impl.LineRecordReader;
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+import org.nd4j.shade.jackson.databind.ObjectMapper;
+
+public class JacksonLineRecordReader extends LineRecordReader {
+
+    private FieldSelection selection;
+    private ObjectMapper mapper;
+
+	public JacksonLineRecordReader(FieldSelection selection, ObjectMapper mapper) {
+		this.selection = selection;
+		this.mapper = mapper;
+	}
+
+    @Override
+    public List<Writable> next() {
+        Text t = (Text) super.next().iterator().next();
+        String val = t.toString();
+        return parseLine(val);
+    }
+    
+    protected List<Writable> parseLine(String line) {
+	    return JacksonReaderUtils.parseRecord(line, selection, mapper);
+    }
+    
+    @Override
+    public List<String> getLabels() {
+        throw new UnsupportedOperationException();
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonLineSequenceRecordReader.java

@@ -0,0 +1,112 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.jackson;
+
+import lombok.SneakyThrows;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.LineIterator;
+import org.datavec.api.records.SequenceRecord;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataURI;
+import org.datavec.api.records.reader.SequenceRecordReader;
+import org.datavec.api.records.reader.impl.FileRecordReader;
+import org.datavec.api.writable.Writable;
+import org.nd4j.shade.jackson.databind.ObjectMapper;
+
+import java.io.*;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public class JacksonLineSequenceRecordReader extends FileRecordReader implements SequenceRecordReader {
+
+    private FieldSelection selection;
+    private ObjectMapper mapper;
+
+    /**
+     *
+     * @param selection Fields to return
+     * @param mapper    Object mapper to use. For example, {@code new ObjectMapper(new JsonFactory())} for JSON
+     */
+    public JacksonLineSequenceRecordReader(FieldSelection selection, ObjectMapper mapper){
+        this.selection = selection;
+        this.mapper = mapper;
+    }
+
+    @Override
+    public List<List<Writable>> sequenceRecord() {
+        return nextSequence().getSequenceRecord();
+    }
+
+    @Override
+    public List<List<Writable>> sequenceRecord(URI uri, DataInputStream dataInputStream) throws IOException {
+        return loadAndClose(dataInputStream);
+    }
+
+    @Override
+    public SequenceRecord nextSequence() {
+        if(!hasNext()){
+            throw new NoSuchElementException("No next element");
+        }
+
+        URI next = locationsIterator.next();
+        List<List<Writable>> out = loadAndClose(streamCreatorFn.apply(next));
+        return new org.datavec.api.records.impl.SequenceRecord(out, new RecordMetaDataURI(next));
+    }
+
+    @Override
+    public SequenceRecord loadSequenceFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return null;
+    }
+
+    @Override
+    public List<SequenceRecord> loadSequenceFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        List<SequenceRecord> out = new ArrayList<>();
+        for(RecordMetaData m : recordMetaDatas){
+            out.add(loadSequenceFromMetaData(m));
+        }
+        return out;
+    }
+
+    @SneakyThrows
+    private List<List<Writable>> loadAndClose(InputStream inputStream) {
+        LineIterator lineIter = null;
+        try {
+            lineIter = IOUtils.lineIterator(new BufferedReader(new InputStreamReader(inputStream)));
+            return load(lineIter);
+        } finally {
+            if (lineIter != null) {
+                lineIter.close();
+            }
+            IOUtils.closeQuietly(inputStream);
+        }
+    }
+
+    private List<List<Writable>> load(Iterator<String> lineIter) {
+        List<List<Writable>> out = new ArrayList<>();
+        while(lineIter.hasNext()){
+            out.add(JacksonReaderUtils.parseRecord(lineIter.next(), selection, mapper));
+        }
+        return out;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonReaderUtils.java

@@ -0,0 +1,95 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.jackson;
+
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+import org.nd4j.shade.jackson.core.type.TypeReference;
+import org.nd4j.shade.jackson.databind.ObjectMapper;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+public class JacksonReaderUtils {
+
+    private static final TypeReference<Map<String, Object>> typeRef = new TypeReference<Map<String, Object>>() {};
+
+    private JacksonReaderUtils(){ }
+
+    public static List<Writable> parseRecord(String line, FieldSelection selection, ObjectMapper mapper){
+        List<Writable> out = new ArrayList<>();
+        List<String[]> paths = selection.getFieldPaths();
+        List<Writable> valueIfMissing = selection.getValueIfMissing();
+        Map<String, Object> map;
+        try {
+            map = mapper.readValue(line, typeRef);
+        } catch (IOException e) {
+            throw new RuntimeException("Error parsing file", e);
+        }
+
+        //Now, extract out values...
+        for (int i = 0; i < paths.size(); i++) {
+            String[] currPath = paths.get(i);
+            String value = null;
+            Map<String, Object> currMap = map;
+            for (int j = 0; j < currPath.length; j++) {
+                if (currMap.containsKey(currPath[j])) {
+                    Object o = currMap.get(currPath[j]);
+                    if (j == currPath.length - 1) {
+                        //Expect to get the final value
+                        if (o instanceof String) {
+                            value = (String) o;
+                        } else if (o instanceof Number) {
+                            value = o.toString();
+                        } else {
+                            throw new IllegalStateException(
+                                    "Expected to find String on path " + Arrays.toString(currPath) + ", found "
+                                            + o.getClass() + " with value " + o);
+                        }
+                    } else {
+                        //Expect to get a map...
+                        if (o instanceof Map) {
+                            currMap = (Map<String, Object>) o;
+                        }
+                    }
+                } else {
+                    //Not found
+                    value = null;
+                    break;
+                }
+            }
+
+            Writable outputWritable;
+            if (value == null) {
+                outputWritable = valueIfMissing.get(i);
+            } else {
+                outputWritable = new Text(value);
+            }
+            out.add(outputWritable);
+        }
+
+        return out;
+    }
+
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonRecordReader.java

@@ -0,0 +1,229 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.jackson;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.io.labels.PathLabelGenerator;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataURI;
+import org.datavec.api.records.reader.BaseRecordReader;
+import org.datavec.api.split.FileSplit;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Writable;
+import org.nd4j.shade.jackson.core.type.TypeReference;
+import org.nd4j.shade.jackson.databind.ObjectMapper;
+
+import java.io.*;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+
+public class JacksonRecordReader extends BaseRecordReader {
+
+    private static final TypeReference<Map<String, Object>> typeRef = new TypeReference<Map<String, Object>>() {};
+
+    private FieldSelection selection;
+    private ObjectMapper mapper;
+    private boolean shuffle;
+    private long rngSeed;
+    private PathLabelGenerator labelGenerator;
+    private int labelPosition;
+    private InputSplit is;
+    private Random r;
+    @Getter @Setter
+    private String charset = StandardCharsets.UTF_8.name(); //Using String as StandardCharsets.UTF_8 is not serializable
+
+    private URI[] uris;
+    private int cursor = 0;
+
+    public JacksonRecordReader(FieldSelection selection, ObjectMapper mapper) {
+        this(selection, mapper, false);
+    }
+
+    public JacksonRecordReader(FieldSelection selection, ObjectMapper mapper, boolean shuffle) {
+        this(selection, mapper, shuffle, System.currentTimeMillis(), null);
+    }
+
+    public JacksonRecordReader(FieldSelection selection, ObjectMapper mapper, boolean shuffle, long rngSeed,
+                    PathLabelGenerator labelGenerator) {
+        this(selection, mapper, shuffle, rngSeed, labelGenerator, -1);
+    }
+
+    public JacksonRecordReader(FieldSelection selection, ObjectMapper mapper, boolean shuffle, long rngSeed,
+                    PathLabelGenerator labelGenerator, int labelPosition) {
+        this.selection = selection;
+        this.mapper = mapper;
+        this.shuffle = shuffle;
+        this.rngSeed = rngSeed;
+        if (shuffle)
+            r = new Random(rngSeed);
+        this.labelGenerator = labelGenerator;
+        this.labelPosition = labelPosition;
+    }
+
+    @Override
+    public void initialize(InputSplit split) throws IOException, InterruptedException {
+        if (split instanceof FileSplit)
+            throw new UnsupportedOperationException("Cannot use JacksonRecordReader with FileSplit");
+        super.initialize(inputSplit);
+        this.uris = split.locations();
+        if (shuffle) {
+            List<URI> list = Arrays.asList(uris);
+            Collections.shuffle(list, r);
+            uris = list.toArray(new URI[uris.length]);
+        }
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        initialize(split);
+    }
+
+    @Override
+    public List<Writable> next() {
+        if (uris == null)
+            throw new IllegalStateException("URIs are null. Not initialized?");
+        if (!hasNext())
+            throw new NoSuchElementException("No next element");
+
+        URI uri = uris[cursor++];
+        invokeListeners(uri);
+        String fileAsString;
+        try (InputStream s = streamCreatorFn.apply(uri)){
+            fileAsString = IOUtils.toString(s, charset);
+        } catch (IOException e) {
+            throw new RuntimeException("Error reading URI file", e);
+        }
+
+        return readValues(uri, fileAsString);
+
+    }
+
+    @Override
+    public boolean hasNext() {
+        return cursor < uris.length;
+    }
+
+    @Override
+    public List<String> getLabels() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void reset() {
+        cursor = 0;
+        if (shuffle) {
+            List<URI> list = Arrays.asList(uris);
+            Collections.shuffle(list, r);
+            uris = list.toArray(new URI[uris.length]);
+        }
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return true;
+    }
+
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        BufferedReader br = new BufferedReader(new InputStreamReader(dataInputStream));
+        StringBuilder sb = new StringBuilder();
+        String line;
+        while ((line = br.readLine()) != null) {
+            sb.append(line).append("\n");
+        }
+
+        return readValues(uri, sb.toString());
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+
+    @Override
+    public void setConf(Configuration conf) {
+
+    }
+
+    @Override
+    public Configuration getConf() {
+        return null;
+    }
+
+
+    private List<Writable> readValues(URI uri, String fileContents) {
+        List<Writable> out = JacksonReaderUtils.parseRecord(fileContents, selection, mapper);
+
+        //Add label - if required
+        if(labelGenerator != null){
+            Writable label = labelGenerator.getLabelForPath(uri);
+            List<String[]> paths = selection.getFieldPaths();
+            if ((labelPosition >= paths.size() || labelPosition == -1)) {
+                //Edge case: might want label as the last value
+                out.add(label);
+            } else {
+                out.add(labelPosition, label);  //Add and shift existing to right
+            }
+        }
+
+        return out;
+    }
+
+    @Override
+    public Record nextRecord() {
+        URI currentURI = uris[cursor];
+        List<Writable> writables = next();
+        RecordMetaData meta = new RecordMetaDataURI(currentURI, JacksonRecordReader.class);
+        return new org.datavec.api.records.impl.Record(writables, meta);
+    }
+
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return loadFromMetaData(Collections.singletonList(recordMetaData)).get(0);
+    }
+
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+
+        List<Record> out = new ArrayList<>();
+        for (RecordMetaData metaData : recordMetaDatas) {
+            URI uri = metaData.getURI();
+
+            String fileAsString;
+            try {
+                fileAsString = FileUtils.readFileToString(new File(uri.toURL().getFile()));
+            } catch (IOException e) {
+                throw new RuntimeException("Error reading URI file", e);
+            }
+
+            List<Writable> writables = readValues(uri, fileAsString);
+            out.add(new org.datavec.api.records.impl.Record(writables, metaData));
+        }
+
+        return out;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/misc/LibSvmRecordReader.java

@@ -0,0 +1,36 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.misc;
+
+import lombok.extern.slf4j.Slf4j;
+import org.datavec.api.conf.Configuration;
+
+@Slf4j
+public class LibSvmRecordReader extends SVMLightRecordReader {
+    public LibSvmRecordReader() {
+        super();
+    }
+
+    @Override
+    public void setConf(Configuration conf) {
+        super.setConf(conf);
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/misc/MatlabRecordReader.java

@@ -0,0 +1,122 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.misc;
+
+
+import org.datavec.api.records.reader.impl.FileRecordReader;
+import org.datavec.api.writable.DoubleWritable;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.StringReader;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class MatlabRecordReader extends FileRecordReader {
+
+    private List<List<Writable>> records = new ArrayList<>();
+    private Iterator<List<Writable>> currIter;
+
+    @Override
+    public boolean hasNext() {
+        return super.hasNext();
+    }
+
+    @Override
+    public List<Writable> next() {
+        //use the current iterator
+        if (currIter != null && currIter.hasNext())
+            return new ArrayList<>(currIter.next());
+        records.clear();
+        //next file
+        List<Writable> next = super.next();
+        String val = next.iterator().next().toString();
+        StringReader reader = new StringReader(val);
+        int c;
+        char chr;
+        StringBuilder fileContent;
+        boolean isComment;
+
+
+        List<Writable> currRecord = new ArrayList<>();
+        fileContent = new StringBuilder();
+        isComment = false;
+        records.add(currRecord);
+        try {
+            // determine number of attributes
+            while ((c = reader.read()) != -1) {
+                chr = (char) c;
+
+                // comment found?
+                if (chr == '%')
+                    isComment = true;
+
+                // end of line reached
+                if ((chr == '\n') || (chr == '\r')) {
+                    isComment = false;
+                    if (fileContent.length() > 0)
+                        currRecord.add(new DoubleWritable(new Double(fileContent.toString())));
+
+                    if (currRecord.size() > 0) {
+                        currRecord = new ArrayList<>();
+                        records.add(currRecord);
+                    }
+                    fileContent = new StringBuilder();
+                    continue;
+                }
+
+                // skip till end of comment line
+                if (isComment)
+                    continue;
+
+                // separator found?
+                if ((chr == '\t') || (chr == ' ')) {
+                    if (fileContent.length() > 0) {
+                        currRecord.add(new DoubleWritable(new Double(fileContent.toString())));
+                        fileContent = new StringBuilder();
+                    }
+                } else {
+                    fileContent.append(chr);
+                }
+            }
+
+            // last number?
+            if (fileContent.length() > 0)
+                currRecord.add(new DoubleWritable(new Double(fileContent.toString())));
+
+
+            currIter = records.iterator();
+
+        } catch (Exception ex) {
+            ex.printStackTrace();
+            throw new IllegalStateException("Unable to determine structure as Matlab ASCII file: " + ex);
+        }
+        throw new IllegalStateException("Strange state detected");
+    }
+
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        throw new UnsupportedOperationException("Reading Matlab data from DataInputStream: not yet implemented");
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/misc/SVMLightRecordReader.java

@@ -0,0 +1,294 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.misc;
+
+import lombok.extern.slf4j.Slf4j;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataLine;
+import org.datavec.api.records.reader.impl.LineRecordReader;
+import org.datavec.api.writable.DoubleWritable;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.IntWritable;
+import org.datavec.api.writable.Writable;
+import org.datavec.api.conf.Configuration;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+@Slf4j
+public class SVMLightRecordReader extends LineRecordReader {
+    /* Configuration options. */
+    public static final String NAME_SPACE = SVMLightRecordReader.class.getName();
+    public static final String NUM_FEATURES = NAME_SPACE + ".numfeatures";
+    public static final String ZERO_BASED_INDEXING = NAME_SPACE + ".zeroBasedIndexing";
+    public static final String ZERO_BASED_LABEL_INDEXING = NAME_SPACE + ".zeroBasedLabelIndexing";
+    public static final String MULTILABEL = NAME_SPACE + ".multilabel";
+    public static final String NUM_LABELS = NAME_SPACE + ".numLabels";
+
+    /* Constants. */
+    public static final String COMMENT_CHAR = "#";
+    public static final String ALLOWED_DELIMITERS = "[ \t]";
+    public static final String PREFERRED_DELIMITER = " ";
+    public static final String FEATURE_DELIMITER = ":";
+    public static final String LABEL_DELIMITER = ",";
+    public static final String QID_PREFIX = "qid";
+
+    /* For convenience */
+    public static final Writable ZERO = new DoubleWritable(0);
+    public static final Writable ONE = new DoubleWritable(1);
+    public static final Writable LABEL_ZERO = new IntWritable(0);
+    public static final Writable LABEL_ONE = new IntWritable(1);
+
+    protected int numFeatures = -1; // number of features
+    protected boolean zeroBasedIndexing = true; /* whether to use zero-based indexing, true is safest
+                                                 * but adds extraneous column if data is not zero indexed
+                                                 */
+    protected boolean zeroBasedLabelIndexing = false; // whether to use zero-based label indexing (NONSTANDARD!)
+    protected boolean appendLabel = true; // whether to append labels to output
+    protected boolean multilabel = false; // whether targets are multilabel
+    protected int numLabels = -1; // number of labels (required for multilabel targets)
+    protected Writable recordLookahead = null;
+
+    // for backwards compatibility
+    public final static String NUM_ATTRIBUTES = NAME_SPACE + ".numattributes";
+
+    public SVMLightRecordReader() {}
+
+    /**
+     * Must be called before attempting to read records.
+     *
+     * @param conf          DataVec configuration
+     * @param split         FileSplit
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        super.initialize(conf, split);
+        setConf(conf);
+    }
+
+    /**
+     * Set configuration.
+     *
+     * @param conf          DataVec configuration
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Override
+    public void setConf(Configuration conf) {
+        super.setConf(conf);
+        numFeatures = conf.getInt(NUM_FEATURES, -1);
+        if (numFeatures < 0)
+            numFeatures = conf.getInt(NUM_ATTRIBUTES, -1);
+        if (numFeatures < 0)
+            throw new UnsupportedOperationException("numFeatures must be set in configuration");
+        appendLabel = conf.getBoolean(APPEND_LABEL, true);
+        multilabel = conf.getBoolean(MULTILABEL, false);
+        zeroBasedIndexing = conf.getBoolean(ZERO_BASED_INDEXING, true);
+        zeroBasedLabelIndexing = conf.getBoolean(ZERO_BASED_LABEL_INDEXING, false);
+        numLabels = conf.getInt(NUM_LABELS, -1);
+        if (multilabel && numLabels < 0)
+            throw new UnsupportedOperationException("numLabels must be set in confirmation for multilabel problems");
+    }
+
+    /**
+     * Helper function to help detect lines that are
+     * commented out. May read ahead and cache a line.
+     *
+     * @return
+     */
+    protected Writable getNextRecord() {
+        Writable w = null;
+        if (recordLookahead != null) {
+            w = recordLookahead;
+            recordLookahead = null;
+        }
+        while (w == null && super.hasNext()) {
+            w = super.next().iterator().next();
+            if (!w.toString().startsWith(COMMENT_CHAR))
+                break;
+            w = null;
+        }
+        return w;
+    }
+
+    @Override
+    public boolean hasNext() {
+        recordLookahead = getNextRecord();
+        return (recordLookahead != null);
+    }
+
+    /**
+     * Return next record as list of Writables.
+     *
+     * @return
+     */
+    @Override
+    public List<Writable> next() {
+        if(numFeatures < 0 && numLabels < 0){
+            throw new IllegalStateException("Cannot get record: setConf(Configuration) has not been called. A setConf " +
+                    "call is rquired to specify the number of features and/or labels in the source dataset");
+        }
+
+
+        Writable w = getNextRecord();
+        if (w == null)
+            throw new NoSuchElementException("No next element found!");
+        String line = w.toString();
+        List<Writable> record = new ArrayList<>(Collections.nCopies(numFeatures, ZERO));
+
+        // Remove trailing comments
+        String commentRegex = ALLOWED_DELIMITERS + "*" + COMMENT_CHAR + ".*$";
+        String[] tokens = line.replaceFirst(commentRegex, "").split(ALLOWED_DELIMITERS);
+
+        // Iterate over feature tokens
+        for (int i = 1; i < tokens.length; i++) {
+            String token = tokens[i];
+            // Split into feature index and value
+            String[] featureTokens = token.split(FEATURE_DELIMITER);
+            if (featureTokens[0].startsWith(QID_PREFIX)) {
+                // Ignore QID entry for now
+            } else {
+                // Parse feature index -- enforce that it's a positive integer
+                int index = -1;
+                try {
+                    index = Integer.parseInt(featureTokens[0]);
+                    if (index < 0)
+                        throw new NumberFormatException("");
+                } catch (NumberFormatException e) {
+                    String msg = String.format("Feature index must be positive integer (found %s)", featureTokens[i].toString());
+                    throw new NumberFormatException(msg);
+                }
+
+                // If not using zero-based indexing, shift all indeces to left by one
+                if (!zeroBasedIndexing) {
+                    if (index == 0)
+                        throw new IndexOutOfBoundsException("Found feature with index " + index + " but not using zero-based indexing");
+                    index--;
+                }
+
+                // Check whether feature index exceeds number of features
+                if (numFeatures >= 0 && index >= numFeatures)
+                    throw new IndexOutOfBoundsException("Found " + (index+1) + " features in record, expected " + numFeatures);
+
+                // Add feature
+                record.set(index, new DoubleWritable(Double.parseDouble(featureTokens[1])));
+            }
+        }
+
+        // If labels should be appended
+        if (appendLabel) {
+            List<Writable> labels = new ArrayList<>();
+
+            // Treat labels as indeces for multilabel binary classification
+            if (multilabel) {
+                labels = new ArrayList<>(Collections.nCopies(numLabels, LABEL_ZERO));
+                if (!tokens[0].equals("")) {
+                    String[] labelTokens = tokens[0].split(LABEL_DELIMITER);
+                    for (int i = 0; i < labelTokens.length; i++) {
+                        // Parse label index -- enforce that it's a positive integer
+                        int index = -1;
+                        try {
+                            index = Integer.parseInt(labelTokens[i]);
+                            if (index < 0)
+                                throw new NumberFormatException("");
+                        } catch (NumberFormatException e) {
+                            String msg = String.format("Multilabel index must be positive integer (found %s)", labelTokens[i].toString());
+                            throw new NumberFormatException(msg);
+                        }
+
+                        // If not using zero-based indexing for labels, shift all indeces to left by one
+                        if (!zeroBasedLabelIndexing) {
+                            if (index == 0)
+                                throw new IndexOutOfBoundsException("Found label with index " + index + " but not using zero-based indexing");
+                            index--;
+                        }
+
+                        // Check whether label index exceeds number of labels
+                        if (numLabels >= 0 && index >= numLabels)
+                            throw new IndexOutOfBoundsException("Found " + (index + 1) + " labels in record, expected " + numLabels);
+
+                        // Add label
+                        labels.set(index, LABEL_ONE);
+                    }
+                }
+            } else {
+                String[] labelTokens = tokens[0].split(LABEL_DELIMITER);
+                int numLabelsFound = labelTokens[0].equals("") ? 0 : labelTokens.length;
+                if (numLabels < 0)
+                    numLabels = numLabelsFound;
+                if (numLabelsFound != numLabels)
+                    throw new IndexOutOfBoundsException("Found " + labelTokens.length + " labels in record, expected " + numLabels);
+                for (int i = 0; i < numLabelsFound; i++) {
+                    try { // Encode label as integer, if possible
+                        labels.add(new IntWritable(Integer.parseInt(labelTokens[i])));
+                    } catch (NumberFormatException e) {
+                        labels.add(new DoubleWritable(Double.parseDouble(labelTokens[i])));
+                    }
+                }
+            }
+
+            // Append labels to record
+            record.addAll(labels);
+        }
+
+        return record;
+    }
+
+    /**
+     * Return next Record.
+     *
+     * @return
+     */
+    @Override
+    public Record nextRecord() {
+        List<Writable> next = next();
+        URI uri = (locations == null || locations.length < 1 ? null : locations[splitIndex]);
+        RecordMetaData meta = new RecordMetaDataLine(this.lineIndex - 1, uri, SVMLightRecordReader.class); //-1 as line number has been incremented already...
+        return new org.datavec.api.records.impl.Record(next, meta);
+    }
+
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        //Here: we are reading a single line from the DataInputStream. How to handle headers?
+        throw new UnsupportedOperationException(
+                "Reading SVMLightRecordReader data from DataInputStream not yet implemented");
+    }
+
+    @Override
+    public void reset() {
+        super.reset();
+        recordLookahead = null;
+    }
+
+    @Override
+    protected void onLocationOpen(URI location) {
+        super.onLocationOpen(location);
+        recordLookahead = null;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/regex/RegexLineRecordReader.java

@@ -0,0 +1,131 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.regex;
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataLine;
+import org.datavec.api.records.reader.impl.LineRecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class RegexLineRecordReader extends LineRecordReader {
+    public final static String SKIP_NUM_LINES = NAME_SPACE + ".skipnumlines";
+
+    private String regex;
+    private int skipNumLines;
+    private Pattern pattern;
+    private int numLinesSkipped;
+    private int currLine = 0;
+
+    public RegexLineRecordReader(String regex, int skipNumLines) {
+        this.regex = regex;
+        this.skipNumLines = skipNumLines;
+        this.pattern = Pattern.compile(regex);
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        super.initialize(conf, split);
+        this.skipNumLines = conf.getInt(SKIP_NUM_LINES, this.skipNumLines);
+    }
+
+    @Override
+    public List<Writable> next() {
+        if (numLinesSkipped < skipNumLines) {
+            for (int i = numLinesSkipped; i < skipNumLines; i++, numLinesSkipped++) {
+                if (!hasNext()) {
+                    return new ArrayList<>();
+                }
+                super.next();
+            }
+        }
+        Text t = (Text) super.next().iterator().next();
+        String val = t.toString();
+        return parseLine(val);
+    }
+
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        Writable w = super.record(uri, dataInputStream).get(0);
+        return parseLine(w.toString());
+    }
+
+    private List<Writable> parseLine(String line) {
+        Matcher m = pattern.matcher(line);
+
+        List<Writable> ret;
+        if (m.matches()) {
+            int count = m.groupCount();
+            ret = new ArrayList<>(count);
+            for (int i = 1; i <= count; i++) { //Note: Matcher.group(0) is the entire sequence; we only care about groups 1 onward
+                ret.add(new Text(m.group(i)));
+            }
+        } else {
+            throw new IllegalStateException("Invalid line: line does not match regex (line #" + currLine + ", regex=\""
+                            + regex + "\"; line=\"" + line + "\"");
+        }
+
+        return ret;
+    }
+
+    @Override
+    public void reset() {
+        super.reset();
+        numLinesSkipped = 0;
+    }
+
+    @Override
+    public Record nextRecord() {
+        List<Writable> next = next();
+        URI uri = (locations == null || locations.length < 1 ? null : locations[splitIndex]);
+        RecordMetaData meta = new RecordMetaDataLine(this.lineIndex - 1, uri, RegexLineRecordReader.class); //-1 as line number has been incremented already...
+        return new org.datavec.api.records.impl.Record(next, meta);
+    }
+
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return loadFromMetaData(Collections.singletonList(recordMetaData)).get(0);
+    }
+
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        List<Record> list = super.loadFromMetaData(recordMetaDatas);
+
+        for (Record r : list) {
+            String line = r.getRecord().get(0).toString();
+            r.setRecord(parseLine(line));
+        }
+
+        return list;
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/regex/RegexSequenceRecordReader.java

@@ -0,0 +1,191 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.regex;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.SequenceRecord;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.metadata.RecordMetaDataURI;
+import org.datavec.api.records.reader.SequenceRecordReader;
+import org.datavec.api.records.reader.impl.FileRecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+import org.nd4j.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.net.URI;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class RegexSequenceRecordReader extends FileRecordReader implements SequenceRecordReader {
+    public static final String SKIP_NUM_LINES = NAME_SPACE + ".skipnumlines";
+    public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
+    public static final LineErrorHandling DEFAULT_ERROR_HANDLING = LineErrorHandling.FailOnInvalid;
+
+    /**Error handling mode: How should invalid lines (i.e., those that don't match the provided regex) be handled?<br>
+     * FailOnInvalid: Throw an IllegalStateException when an invalid line is found<br>
+     * SkipInvalid: Skip invalid lines (quietly, with no warning)<br>
+     * SkipInvalidWithWarning: Skip invalid lines, but log a warning<br>
+     */
+    public enum LineErrorHandling {
+        FailOnInvalid, SkipInvalid, SkipInvalidWithWarning
+    }
+
+    public static final Logger LOG = LoggerFactory.getLogger(RegexSequenceRecordReader.class);
+
+    private String regex;
+    private int skipNumLines;
+    private Pattern pattern;
+    private transient Charset charset;
+    private LineErrorHandling errorHandling;
+
+    public RegexSequenceRecordReader(String regex, int skipNumLines) {
+        this(regex, skipNumLines, DEFAULT_CHARSET, DEFAULT_ERROR_HANDLING);
+    }
+
+    public RegexSequenceRecordReader(String regex, int skipNumLines, Charset encoding,
+                    LineErrorHandling errorHandling) {
+        this.regex = regex;
+        this.skipNumLines = skipNumLines;
+        this.pattern = Pattern.compile(regex);
+        this.charset = encoding;
+        this.errorHandling = errorHandling;
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        super.initialize(conf, split);
+        this.skipNumLines = conf.getInt(SKIP_NUM_LINES, this.skipNumLines);
+    }
+
+    public List<List<Writable>> sequenceRecord() {
+        return nextSequence().getSequenceRecord();
+    }
+
+    @Override
+    public List<List<Writable>> sequenceRecord(URI uri, DataInputStream dataInputStream) throws IOException {
+        String fileContents = IOUtils.toString(new BufferedInputStream(dataInputStream), charset.name());
+        return loadSequence(fileContents, uri);
+    }
+
+    private List<List<Writable>> loadSequence(String fileContents, URI uri) {
+        String[] lines = fileContents.split("(\r\n)|\n"); //TODO this won't work if regex allows for a newline
+
+        int numLinesSkipped = 0;
+        List<List<Writable>> out = new ArrayList<>();
+        int lineCount = 0;
+        for (String line : lines) {
+            lineCount++;
+            if (numLinesSkipped < skipNumLines) {
+                numLinesSkipped++;
+                continue;
+            }
+            //Split line using regex matcher
+            Matcher m = pattern.matcher(line);
+            List<Writable> timeStep;
+            if (m.matches()) {
+                int count = m.groupCount();
+                timeStep = new ArrayList<>(count);
+                for (int i = 1; i <= count; i++) { //Note: Matcher.group(0) is the entire sequence; we only care about groups 1 onward
+                    timeStep.add(new Text(m.group(i)));
+                }
+            } else {
+                switch (errorHandling) {
+                    case FailOnInvalid:
+                        throw new IllegalStateException(
+                                        "Invalid line: line does not match regex (line #" + lineCount + ", uri=\"" + uri
+                                                        + "\"), " + "\", regex=" + regex + "\"; line=\"" + line + "\"");
+                    case SkipInvalid:
+                        continue;
+                    case SkipInvalidWithWarning:
+                        String warnMsg = "Skipping invalid line: line does not match regex (line #" + lineCount
+                                        + ", uri=\"" + uri + "\"), " + "\"; line=\"" + line + "\"";
+                        LOG.warn(warnMsg);
+                        continue;
+                    default:
+                        throw new RuntimeException("Unknown error handling mode: " + errorHandling);
+                }
+            }
+            out.add(timeStep);
+        }
+
+        return out;
+    }
+
+    @Override
+    public void reset() {
+        super.reset();
+    }
+
+    @Override
+    public SequenceRecord nextSequence() {
+        Preconditions.checkState(hasNext(), "No next element available");
+        URI next = locationsIterator.next();
+
+        String fileContents;
+        try (InputStream s = streamCreatorFn.apply(next)){
+            fileContents = IOUtils.toString(s, charset);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        List<List<Writable>> sequence = loadSequence(fileContents, next);
+        return new org.datavec.api.records.impl.SequenceRecord(sequence, new RecordMetaDataURI(next, RegexSequenceRecordReader.class));
+    }
+
+    @Override
+    public SequenceRecord loadSequenceFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return loadSequenceFromMetaData(Collections.singletonList(recordMetaData)).get(0);
+    }
+
+    @Override
+    public List<SequenceRecord> loadSequenceFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        List<SequenceRecord> out = new ArrayList<>();
+        for (RecordMetaData meta : recordMetaDatas) {
+            File next = new File(meta.getURI());
+            URI uri = next.toURI();
+            String fileContents = FileUtils.readFileToString(next, charset.name());
+            List<List<Writable>> sequence = loadSequence(fileContents, uri);
+            out.add(new org.datavec.api.records.impl.SequenceRecord(sequence, meta));
+        }
+        return out;
+    }
+
+    private void readObject(ObjectInputStream ois) throws ClassNotFoundException, IOException {
+        ois.defaultReadObject();
+        String s = ois.readUTF();
+        charset = Charset.forName(s);
+    }
+
+    private void writeObject(ObjectOutputStream oos) throws IOException {
+        oos.defaultWriteObject();
+        oos.writeUTF(charset.name());
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReader.java

@@ -0,0 +1,282 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.transform;
+
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.listener.RecordListener;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.reader.RecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.transform.TransformProcess;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public class TransformProcessRecordReader implements RecordReader {
+    protected RecordReader recordReader;
+    protected TransformProcess transformProcess;
+
+    //Cached/prefetched values, in case of filtering
+    protected Record next;
+
+    public TransformProcessRecordReader(RecordReader recordReader, TransformProcess transformProcess){
+        this.recordReader = recordReader;
+        this.transformProcess = transformProcess;
+    }
+
+    /**
+     * Called once at initialization.
+     *
+     * @param split the split that defines the range of records to read
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Override
+    public void initialize(InputSplit split) throws IOException, InterruptedException {
+        recordReader.initialize(split);
+    }
+
+    /**
+     * Called once at initialization.
+     *
+     * @param conf  a configuration for initialization
+     * @param split the split that defines the range of records to read
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        recordReader.initialize(conf, split);
+    }
+
+    @Override
+    public boolean batchesSupported() {
+        return true;
+    }
+
+    @Override
+    public List<List<Writable>> next(int num) {
+        if(!hasNext())
+            throw new NoSuchElementException("No next element");
+
+        List<List<Writable>> out = new ArrayList<>();
+        for( int i=0; i<num && hasNext(); i++ ){
+            out.add(next());
+        }
+        return out;
+    }
+
+    /**
+     * Get the next record
+     *
+     * @return
+     */
+    @Override
+    public List<Writable> next() {
+        if(!hasNext()){ //Also triggers prefetch
+            throw new NoSuchElementException("No next element");
+        }
+        List<Writable> out = next.getRecord();
+        next = null;
+        return out;
+    }
+
+    /**
+     * Whether there are anymore records
+     *
+     * @return
+     */
+    @Override
+    public boolean hasNext() {
+        if(next != null){
+            return true;
+        }
+        if(!recordReader.hasNext()){
+            return false;
+        }
+
+        //Prefetch, until we find one that isn't filtered out - or we run out of data
+        while(next == null && recordReader.hasNext()){
+            Record r = recordReader.nextRecord();
+            List<Writable> temp = transformProcess.execute(r.getRecord());
+            if(temp == null){
+                continue;
+            }
+            next = new org.datavec.api.records.impl.Record(temp, r.getMetaData());
+        }
+
+        return next != null;
+    }
+
+    /**
+     * List of label strings
+     *
+     * @return
+     */
+    @Override
+    public List<String> getLabels() {
+        return recordReader.getLabels();
+    }
+
+    /**
+     * Reset record reader iterator
+     *
+     * @return
+     */
+    @Override
+    public void reset() {
+        next = null;
+        recordReader.reset();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return recordReader.resetSupported();
+    }
+
+    /**
+     * Load the record from the given DataInputStream
+     * Unlike {@link #next()} the internal state of the RecordReader is not modified
+     * Implementations of this method should not close the DataInputStream
+     *
+     * @param uri
+     * @param dataInputStream
+     * @throws IOException if error occurs during reading from the input stream
+     */
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        return transformProcess.execute(recordReader.record(uri, dataInputStream));
+    }
+
+    /**
+     * Similar to {@link #next()}, but returns a {@link Record} object, that may include metadata such as the source
+     * of the data
+     *
+     * @return next record
+     */
+    @Override
+    public Record nextRecord() {
+        if(!hasNext()){ //Also triggers prefetch
+            throw new NoSuchElementException("No next element");
+        }
+        Record toRet = next;
+        next = null;
+        return toRet;
+    }
+
+    /**
+     * Load a single record from the given {@link RecordMetaData} instance<br>
+     * Note: that for data that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to
+     * load multiple records at once using {@link #loadFromMetaData(List)}
+     *
+     * @param recordMetaData Metadata for the record that we want to load from
+     * @return Single record for the given RecordMetaData instance
+     * @throws IOException If I/O error occurs during loading
+     */
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        return recordReader.loadFromMetaData(recordMetaData);
+    }
+
+    /**
+     * Load multiple records from the given a list of {@link RecordMetaData} instances<br>
+     *
+     * @param recordMetaDatas Metadata for the records that we want to load from
+     * @return Multiple records for the given RecordMetaData instances
+     * @throws IOException If I/O error occurs during loading
+     */
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        return recordReader.loadFromMetaData(recordMetaDatas);
+    }
+
+    /**
+     * Get the record listeners for this record reader.
+     */
+    @Override
+    public List<RecordListener> getListeners() {
+        return recordReader.getListeners();
+    }
+
+    /**
+     * Set the record listeners for this record reader.
+     *
+     * @param listeners
+     */
+    @Override
+    public void setListeners(RecordListener... listeners) {
+        recordReader.setListeners(listeners);
+    }
+
+    /**
+     * Set the record listeners for this record reader.
+     *
+     * @param listeners
+     */
+    @Override
+    public void setListeners(Collection<RecordListener> listeners) {
+        recordReader.setListeners(listeners);
+
+    }
+
+    /**
+     * Closes this stream and releases any system resources associated
+     * with it. If the stream is already closed then invoking this
+     * method has no effect.
+     * <p>
+     * <p> As noted in {@link AutoCloseable#close()}, cases where the
+     * close may fail require careful attention. It is strongly advised
+     * to relinquish the underlying resources and to internally
+     * <em>mark</em> the {@code Closeable} as closed, prior to throwing
+     * the {@code IOException}.
+     *
+     * @throws IOException if an I/O error occurs
+     */
+    @Override
+    public void close() throws IOException {
+        recordReader.close();
+    }
+
+    /**
+     * Set the configuration to be used by this object.
+     *
+     * @param conf
+     */
+    @Override
+    public void setConf(Configuration conf) {
+
+    }
+
+    /**
+     * Return the configuration used by this object.
+     */
+    @Override
+    public Configuration getConf() {
+        return recordReader.getConf();
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessSequenceRecordReader.java

@@ -0,0 +1,314 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.reader.impl.transform;
+
+import lombok.AllArgsConstructor;
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.SequenceRecord;
+import org.datavec.api.records.listener.RecordListener;
+import org.datavec.api.records.metadata.RecordMetaData;
+import org.datavec.api.records.reader.SequenceRecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.transform.TransformProcess;
+import org.datavec.api.writable.Writable;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Collection;
+import java.util.List;
+
+@AllArgsConstructor
+public class TransformProcessSequenceRecordReader implements SequenceRecordReader {
+
+    protected SequenceRecordReader sequenceRecordReader;
+    protected TransformProcess transformProcess;
+
+
+
+    /**
+     * Set the configuration to be used by this object.
+     *
+     * @param conf
+     */
+    @Override
+    public void setConf(Configuration conf) {
+        sequenceRecordReader.setConf(conf);
+    }
+
+    /**
+     * Return the configuration used by this object.
+     */
+    @Override
+    public Configuration getConf() {
+        return sequenceRecordReader.getConf();
+    }
+
+    /**
+     * Returns a sequence record.
+     *
+     * @return a sequence of records
+     */
+    @Override
+    public List<List<Writable>> sequenceRecord() {
+        return transformProcess.executeSequence(sequenceRecordReader.sequenceRecord());
+    }
+
+    @Override
+    public boolean batchesSupported() {
+        return false;
+    }
+
+    @Override
+    public List<List<Writable>> next(int num) {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Load a sequence record from the given DataInputStream
+     * Unlike {@link #next()} the internal state of the RecordReader is not modified
+     * Implementations of this method should not close the DataInputStream
+     *
+     * @param uri
+     * @param dataInputStream
+     * @throws IOException if error occurs during reading from the input stream
+     */
+    @Override
+    public List<List<Writable>> sequenceRecord(URI uri, DataInputStream dataInputStream) throws IOException {
+        return transformProcess.executeSequence(sequenceRecordReader.sequenceRecord(uri, dataInputStream));
+    }
+
+    /**
+     * Similar to {@link #sequenceRecord()}, but returns a {@link Record} object, that may include metadata such as the source
+     * of the data
+     *
+     * @return next sequence record
+     */
+    @Override
+    public SequenceRecord nextSequence() {
+        SequenceRecord next = sequenceRecordReader.nextSequence();
+        next.setSequenceRecord(transformProcess.executeSequence(next.getSequenceRecord()));
+        return next;
+    }
+
+    /**
+     * Load a single sequence record from the given {@link RecordMetaData} instance<br>
+     * Note: that for data that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to
+     * load multiple records at once using {@link #loadSequenceFromMetaData(List)}
+     *
+     * @param recordMetaData Metadata for the sequence record that we want to load from
+     * @return Single sequence record for the given RecordMetaData instance
+     * @throws IOException If I/O error occurs during loading
+     */
+    @Override
+    public SequenceRecord loadSequenceFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        SequenceRecord next = sequenceRecordReader.loadSequenceFromMetaData(recordMetaData);
+        next.setSequenceRecord(transformProcess.executeSequence(next.getSequenceRecord()));
+        return next;
+    }
+
+    /**
+     * Load multiple sequence records from the given a list of {@link RecordMetaData} instances<br>
+     *
+     * @param recordMetaDatas Metadata for the records that we want to load from
+     * @return Multiple sequence record for the given RecordMetaData instances
+     * @throws IOException If I/O error occurs during loading
+     */
+    @Override
+    public List<SequenceRecord> loadSequenceFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        return null;
+    }
+
+    /**
+     * Called once at initialization.
+     *
+     * @param split the split that defines the range of records to read
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Override
+    public void initialize(InputSplit split) throws IOException, InterruptedException {
+
+    }
+
+    /**
+     * Called once at initialization.
+     *
+     * @param conf  a configuration for initialization
+     * @param split the split that defines the range of records to read
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+
+    }
+
+    /**
+     * Get the next record
+     *
+     * @return
+     */
+    @Override
+    public List<Writable> next() {
+        return transformProcess.execute(sequenceRecordReader.next());
+    }
+
+    /**
+     * Whether there are anymore records
+     *
+     * @return
+     */
+    @Override
+    public boolean hasNext() {
+        return sequenceRecordReader.hasNext();
+    }
+
+    /**
+     * List of label strings
+     *
+     * @return
+     */
+    @Override
+    public List<String> getLabels() {
+        return sequenceRecordReader.getLabels();
+    }
+
+    /**
+     * Reset record reader iterator
+     *
+     * @return
+     */
+    @Override
+    public void reset() {
+        sequenceRecordReader.reset();
+    }
+
+    @Override
+    public boolean resetSupported() {
+        return sequenceRecordReader.resetSupported();
+    }
+
+    /**
+     * Load the record from the given DataInputStream
+     * Unlike {@link #next()} the internal state of the RecordReader is not modified
+     * Implementations of this method should not close the DataInputStream
+     *
+     * @param uri
+     * @param dataInputStream
+     * @throws IOException if error occurs during reading from the input stream
+     */
+    @Override
+    public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
+        return transformProcess.execute(sequenceRecordReader.record(uri, dataInputStream));
+    }
+
+    /**
+     * Similar to {@link #next()}, but returns a {@link Record} object, that may include metadata such as the source
+     * of the data
+     *
+     * @return next record
+     */
+    @Override
+    public Record nextRecord() {
+        Record next = sequenceRecordReader.nextRecord();
+        next.setRecord(transformProcess.execute(next.getRecord()));
+        return next;
+    }
+
+    /**
+     * Load a single record from the given {@link RecordMetaData} instance<br>
+     * Note: that for data that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to
+     * load multiple records at once using {@link #loadFromMetaData(List)}
+     *
+     * @param recordMetaData Metadata for the record that we want to load from
+     * @return Single record for the given RecordMetaData instance
+     * @throws IOException If I/O error occurs during loading
+     */
+    @Override
+    public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
+        Record load = sequenceRecordReader.loadFromMetaData(recordMetaData);
+        load.setRecord(transformProcess.execute(load.getRecord()));
+        return load;
+    }
+
+    /**
+     * Load multiple records from the given a list of {@link RecordMetaData} instances<br>
+     *
+     * @param recordMetaDatas Metadata for the records that we want to load from
+     * @return Multiple records for the given RecordMetaData instances
+     * @throws IOException If I/O error occurs during loading
+     */
+    @Override
+    public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
+        List<Record> records = sequenceRecordReader.loadFromMetaData(recordMetaDatas);
+        for (Record record : records)
+            record.setRecord(transformProcess.execute(record.getRecord()));
+        return records;
+    }
+
+    /**
+     * Get the record listeners for this record reader.
+     */
+    @Override
+    public List<RecordListener> getListeners() {
+        return sequenceRecordReader.getListeners();
+    }
+
+    /**
+     * Set the record listeners for this record reader.
+     *
+     * @param listeners
+     */
+    @Override
+    public void setListeners(RecordListener... listeners) {
+        sequenceRecordReader.setListeners(listeners);
+    }
+
+    /**
+     * Set the record listeners for this record reader.
+     *
+     * @param listeners
+     */
+    @Override
+    public void setListeners(Collection<RecordListener> listeners) {
+        sequenceRecordReader.setListeners(listeners);
+    }
+
+    /**
+     * Closes this stream and releases any system resources associated
+     * with it. If the stream is already closed then invoking this
+     * method has no effect.
+     * <p>
+     * <p> As noted in {@link AutoCloseable#close()}, cases where the
+     * close may fail require careful attention. It is strongly advised
+     * to relinquish the underlying resources and to internally
+     * <em>mark</em> the {@code Closeable} as closed, prior to throwing
+     * the {@code IOException}.
+     *
+     * @throws IOException if an I/O error occurs
+     */
+    @Override
+    public void close() throws IOException {
+        sequenceRecordReader.close();
+    }
+}

datavec/datavec-api/src/main/java/org/datavec/api/records/writer/RecordWriter.java

@@ -0,0 +1,80 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.api.records.writer;
+
+
+import org.datavec.api.conf.Configurable;
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.split.partition.PartitionMetaData;
+import org.datavec.api.split.partition.Partitioner;
+import org.datavec.api.writable.Writable;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+public interface RecordWriter extends Closeable, Configurable {
+    String APPEND = "org.datavec.api.record.writer.append";
+
+
+    /**
+     * Returns true if this record writer
+     * supports efficient batch writing using {@link #writeBatch(List)}
+     * @return
+     */
+    boolean supportsBatch();
+    /**
+     * Initialize a record writer with the given input split
+     * @param inputSplit the input split to initialize with
+     * @param partitioner
+     */
+    void initialize(InputSplit inputSplit, Partitioner partitioner) throws Exception;
+
+    /**
+     * Initialize the record reader with the given configuration
+     * and {@link InputSplit}
+     * @param configuration the configuration to iniailize with
+     * @param split the split to use
+     * @param partitioner
+     */
+    void initialize(Configuration configuration, InputSplit split, Partitioner partitioner) throws Exception;
+
+    /**
+     * Write a record
+     * @param record the record to write
+     */
+    PartitionMetaData write(List<Writable> record) throws IOException;
+
+
+    /**
+     * Write a batch of records
+     * @param batch the batch to write
+     */
+    PartitionMetaData writeBatch(List<List<Writable>> batch) throws IOException;
+
+
+    /**
+     * Close the recod reader
+     */
+    void close();
+
+}