chore: import upstream snapshot with attribution

datavec/datavec-excel/pom.xml

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ /* ******************************************************************************
+  ~  *
+  ~  *
+  ~  * This program and the accompanying materials are made available under the
+  ~  * terms of the Apache License, Version 2.0 which is available at
+  ~  * https://www.apache.org/licenses/LICENSE-2.0.
+  ~  *
+  ~  *  See the NOTICE file distributed with this work for additional
+  ~  *  information regarding copyright ownership.
+  ~  * Unless required by applicable law or agreed to in writing, software
+  ~  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  ~  * License for the specific language governing permissions and limitations
+  ~  * under the License.
+  ~  *
+  ~  * SPDX-License-Identifier: Apache-2.0
+  ~  ******************************************************************************/
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.eclipse.deeplearning4j</groupId>
+        <artifactId>datavec-parent</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>datavec-excel</artifactId>
+
+    <name>datavec-excel</name>
+
+    <properties>
+        <module.name>datavec.excel</module.name>
+    </properties>
+
+    <build>
+        <plugins>
+         <!--   <plugin>
+                <groupId>org.moditect</groupId>
+                <artifactId>moditect-maven-plugin</artifactId>
+            </plugin>-->
+        </plugins>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.eclipse.deeplearning4j</groupId>
+            <artifactId>datavec-api</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.poi/poi -->
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi</artifactId>
+            <version>${poi.version}</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi-ooxml</artifactId>
+            <version>${poi.version}</version>
+        </dependency>
+
+    </dependencies>
+
+
+</project>

datavec/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordReader.java

@@ -0,0 +1,198 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.poi.excel;
+
+import org.apache.poi.ss.usermodel.*;
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.Record;
+import org.datavec.api.records.metadata.RecordMetaDataIndex;
+import org.datavec.api.records.reader.impl.FileRecordReader;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.writable.BooleanWritable;
+import org.datavec.api.writable.DoubleWritable;
+import org.datavec.api.writable.Text;
+import org.datavec.api.writable.Writable;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public class ExcelRecordReader extends FileRecordReader {
+    //originally from CSVRecordReader
+    private boolean skippedLines = false;
+    protected int skipNumLines = 0;
+    public final static String SKIP_NUM_LINES = NAME_SPACE + ".skipnumlines";
+
+    private Iterator<Sheet> sheetIterator;
+    private Iterator<Row> rows;
+    // Create a DataFormatter to format and get each cell's value as String
+    private DataFormatter dataFormatter = new DataFormatter();
+    private Workbook currWorkBook;
+    //we should ensure that the number of columns is consistent across all worksheets
+    private int numColumns = -1;
+
+    /**
+     * Skip skipNumLines number of lines
+     * @param skipNumLines the number of lines to skip
+     */
+    public ExcelRecordReader(int skipNumLines) {
+        this.skipNumLines = skipNumLines;
+    }
+
+
+
+    public ExcelRecordReader() {
+        this(0);
+    }
+
+    @Override
+    public boolean hasNext() {
+        if (!skipLines())
+            throw new NoSuchElementException("No next element found!");
+        return skipLines() && super.hasNext() ||
+                sheetIterator != null && sheetIterator.hasNext()
+                || rows != null && rows.hasNext();
+    }
+
+
+    private boolean skipLines() {
+        if (!skippedLines && skipNumLines > 0) {
+            for (int i = 0; i < skipNumLines; i++) {
+                if (!super.hasNext()) {
+                    return false;
+                }
+                super.next();
+            }
+            skippedLines = true;
+        }
+        return true;
+    }
+
+    @Override
+    public List<Writable> next() {
+        return nextRecord().getRecord();
+    }
+
+    @Override
+    public Record nextRecord(){
+        //start at top tracking rows
+        if(rows != null && rows.hasNext()) {
+            Row currRow = rows.next();
+            List<Writable> ret = new ArrayList<>(currRow.getLastCellNum());
+            for(Cell cell: currRow) {
+                String cellValue = dataFormatter.formatCellValue(cell);
+                ret.add(new Text(cellValue));
+            }
+            Record record = new org.datavec.api.records.impl.Record(ret,
+                                    new RecordMetaDataIndex(
+                                            currRow.getRowNum(),
+                                            super.currentUri,
+                                            ExcelRecordReader.class));
+            return record;
+        }
+        // next track sheets
+        else if(sheetIterator != null && sheetIterator.hasNext()) {
+            Sheet sheet = sheetIterator.next();
+            rows = sheet.rowIterator();
+            Row currRow = rows.next();
+            Record record = new org.datavec.api.records.impl.Record(rowToRecord(currRow),
+                                new RecordMetaDataIndex(
+                                    currRow.getRowNum(),
+                                    super.currentUri,
+                                    ExcelRecordReader.class));
+            return record;
+
+        }
+
+
+        //finally extract workbooks from files and iterate over those starting again at top
+        try(InputStream is = streamCreatorFn.apply(super.locationsIterator.next())) {
+            // Creating a Workbook from an Excel file (.xls or .xlsx)
+            try {
+                if (currWorkBook != null) {
+                    currWorkBook.close();
+                }
+
+                this.currWorkBook = WorkbookFactory.create(is);
+                this.sheetIterator = currWorkBook.sheetIterator();
+                Sheet sheet = sheetIterator.next();
+                rows = sheet.rowIterator();
+                Row currRow = rows.next();
+                Record record = new org.datavec.api.records.impl.Record(rowToRecord(currRow),
+                        new RecordMetaDataIndex(
+                                currRow.getRowNum(),
+                                super.currentUri,
+                                ExcelRecordReader.class));
+                return record;
+
+            } catch (Exception e) {
+                throw new IllegalStateException("Error processing row", e);
+            }
+        } catch (IOException e){
+            throw new RuntimeException("Error reading from stream", e);
+        }
+
+    }
+
+    @Override
+    public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
+        super.initialize(conf, split);
+        this.skipNumLines = conf.getInt(SKIP_NUM_LINES,0);
+    }
+
+    @Override
+    public void reset() {
+        super.reset();
+        skippedLines = false;
+    }
+
+
+
+    private List<Writable> rowToRecord(Row currRow) {
+        if(numColumns < 0) {
+            numColumns = currRow.getLastCellNum();
+        }
+
+        if(currRow.getLastCellNum() != numColumns) {
+            throw new IllegalStateException("Invalid number of columns for row. First number of columns found was " + numColumns + " but row " + currRow.getRowNum() + " was " + currRow.getLastCellNum());
+        }
+
+        List<Writable> ret = new ArrayList<>(currRow.getLastCellNum());
+        for(Cell cell: currRow) {
+            String cellValue = dataFormatter.formatCellValue(cell);
+            switch(cell.getCellType()) {
+                case BLANK: ret.add(new Text("")); break;
+                case STRING: ret.add(new Text("")); break;
+                case BOOLEAN: ret.add(new BooleanWritable(Boolean.valueOf(cellValue))); break;
+                case NUMERIC: ret.add(new DoubleWritable(Double.parseDouble(cellValue))); break;
+                default: ret.add(new Text(cellValue));
+            }
+        }
+
+        return ret;
+
+    }
+
+
+}

datavec/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordWriter.java

@@ -0,0 +1,177 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  *  See the NOTICE file distributed with this work for additional
+ *  *  information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+package org.datavec.poi.excel;
+
+import org.apache.poi.hssf.usermodel.HSSFWorkbook;
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import org.datavec.api.conf.Configuration;
+import org.datavec.api.records.writer.impl.FileRecordWriter;
+import org.datavec.api.split.InputSplit;
+import org.datavec.api.split.partition.PartitionMetaData;
+import org.datavec.api.split.partition.Partitioner;
+import org.datavec.api.writable.*;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+public class ExcelRecordWriter extends FileRecordWriter {
+
+    public final static String WORKSHEET_NAME = "org.datavec.poi.excel.worksheet.name";
+    public final static String FILE_TYPE = "org.datavec.poi.excel.type";
+
+
+    public final static String DEFAULT_FILE_TYPE = "xlsx";
+    public final static String DEFAULT_WORKSHEET_NAME = "datavec-worksheet";
+
+
+    private String workBookName = DEFAULT_WORKSHEET_NAME;
+    private String fileTypeToUse = DEFAULT_FILE_TYPE;
+
+    private Sheet sheet;
+    private Workbook workbook;
+
+
+
+    private void createRow(int rowNum,int numCols,List<Writable> value) {
+        // Create a Row
+        Row headerRow = sheet.createRow(rowNum);
+        int col = 0;
+        for(Writable writable : value) {
+            // Creating cells
+            Cell cell = headerRow.createCell(col++);
+            setValueForCell(cell,writable);
+
+
+        }
+
+        // Resize all columns to fit the content size
+        for(int i = 0; i < numCols; i++) {
+            sheet.autoSizeColumn(i);
+        }
+    }
+
+    private void setValueForCell(Cell cell,Writable value) {
+        if(value instanceof DoubleWritable || value instanceof LongWritable || value instanceof FloatWritable || value instanceof IntWritable) {
+            cell.setCellValue(value.toDouble());
+        }
+        else if(value instanceof BooleanWritable) {
+            cell.setCellValue(((BooleanWritable) value).get());
+        }
+        else if(value instanceof Text) {
+            cell.setCellValue(value.toString());
+        }
+
+    }
+
+
+    @Override
+    public boolean supportsBatch() {
+        return true;
+    }
+
+    @Override
+    public void initialize(InputSplit inputSplit, Partitioner partitioner) throws Exception {
+        this.conf = new Configuration();
+        this.partitioner = partitioner;
+        partitioner.init(inputSplit);
+        out = new DataOutputStream(partitioner.currentOutputStream());
+        initPoi();
+
+
+    }
+
+    private void initPoi()  {
+        if(fileTypeToUse.equals("xlsx"))
+            workbook = new XSSFWorkbook();
+        else {
+            //xls
+            workbook = new HSSFWorkbook();
+        }
+
+        this.sheet = workbook.createSheet(workBookName);
+
+
+    }
+
+    @Override
+    public void initialize(Configuration configuration, InputSplit split, Partitioner partitioner) throws Exception {
+        this.workBookName = configuration.get(WORKSHEET_NAME,DEFAULT_WORKSHEET_NAME);
+        this.fileTypeToUse = configuration.get(FILE_TYPE,DEFAULT_FILE_TYPE);
+        this.conf = configuration;
+        partitioner.init(split);
+        out = new DataOutputStream(partitioner.currentOutputStream());
+        initPoi();
+    }
+
+    @Override
+    public PartitionMetaData write(List<Writable> record) throws IOException {
+        createRow(partitioner.numRecordsWritten(),record.size(),record);
+        reinitIfNecessary();
+        return PartitionMetaData.builder().numRecordsUpdated(1).build();
+    }
+
+    @Override
+    public PartitionMetaData writeBatch(List<List<Writable>> batch) throws IOException {
+        int numSoFar = 0;
+        for (List<Writable> record : batch) {
+            createRow(partitioner.numRecordsWritten() + numSoFar, record.size(), record);
+            reinitIfNecessary();
+            numSoFar++;
+        }
+
+        return PartitionMetaData.builder().numRecordsUpdated(batch.size()).build();
+    }
+
+    private void reinitIfNecessary() throws IOException {
+        if(partitioner.needsNewPartition()) {
+            workbook.write(out);
+            out.flush();
+            out.close();
+            workbook.close();
+            initPoi();
+            this.out = new DataOutputStream(partitioner.openNewStream());
+        }
+    }
+
+    @Override
+    public void close() {
+        if(workbook != null) {
+            try {
+                if(out != null) {
+                    workbook.write(out);
+                    out.flush();
+                    out.close();
+                }
+
+                workbook.close();
+
+            } catch (IOException e) {
+                throw new IllegalStateException(e);
+            }
+        }
+    }
+
+}