chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,66 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ /* ******************************************************************************
|
||||
~ *
|
||||
~ *
|
||||
~ * This program and the accompanying materials are made available under the
|
||||
~ * terms of the Apache License, Version 2.0 which is available at
|
||||
~ * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
~ *
|
||||
~ * See the NOTICE file distributed with this work for additional
|
||||
~ * information regarding copyright ownership.
|
||||
~ * Unless required by applicable law or agreed to in writing, software
|
||||
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
~ * License for the specific language governing permissions and limitations
|
||||
~ * under the License.
|
||||
~ *
|
||||
~ * SPDX-License-Identifier: Apache-2.0
|
||||
~ ******************************************************************************/
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>deeplearning4j-data</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>deeplearning4j-datavec-iterators</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>deeplearning4j-datavec-iterators</name>
|
||||
|
||||
<properties>
|
||||
<module.name>deeplearning4j.datavec.iterators</module.name>
|
||||
</properties>
|
||||
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.moditect</groupId>
|
||||
<artifactId>moditect-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>datavec-api</artifactId>
|
||||
<version>${datavec.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>nd4j-api</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
</project>
|
||||
+580
@@ -0,0 +1,580 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.deeplearning4j.datasets.datavec;
|
||||
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.NonNull;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.datavec.api.io.WritableConverter;
|
||||
import org.datavec.api.io.converters.SelfWritableConverter;
|
||||
import org.datavec.api.records.Record;
|
||||
import org.datavec.api.records.metadata.RecordMetaData;
|
||||
import org.datavec.api.records.metadata.RecordMetaDataComposableMap;
|
||||
import org.datavec.api.records.reader.RecordReader;
|
||||
import org.datavec.api.records.reader.SequenceRecordReader;
|
||||
import org.datavec.api.records.reader.impl.ConcatenatingRecordReader;
|
||||
import org.datavec.api.records.reader.impl.collection.CollectionRecordReader;
|
||||
import org.datavec.api.writable.Writable;
|
||||
import org.nd4j.common.base.Preconditions;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.dataset.DataSet;
|
||||
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
|
||||
import org.nd4j.linalg.dataset.api.MultiDataSet;
|
||||
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
|
||||
import org.nd4j.linalg.factory.Nd4j;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
@Slf4j
|
||||
public class RecordReaderDataSetIterator implements DataSetIterator {
|
||||
private static final String READER_KEY = "reader";
|
||||
@Getter
|
||||
protected RecordReader recordReader;
|
||||
protected WritableConverter converter;
|
||||
protected int batchSize = 10;
|
||||
protected int maxNumBatches = -1;
|
||||
protected int batchNum = 0;
|
||||
protected int labelIndex = -1;
|
||||
protected int labelIndexTo = -1;
|
||||
protected int numPossibleLabels = -1;
|
||||
protected Iterator<List<Writable>> sequenceIter;
|
||||
protected DataSet last;
|
||||
protected boolean useCurrent = false;
|
||||
protected boolean regression = false;
|
||||
@Getter
|
||||
protected DataSetPreProcessor preProcessor;
|
||||
|
||||
@Getter
|
||||
private boolean collectMetaData = false;
|
||||
|
||||
private RecordReaderMultiDataSetIterator underlying;
|
||||
private boolean underlyingIsDisjoint;
|
||||
|
||||
/**
|
||||
* Constructor for classification, where:<br>
|
||||
* (a) the label index is assumed to be the very last Writable/column, and<br>
|
||||
* (b) the number of classes is inferred from RecordReader.getLabels()<br>
|
||||
* Note that if RecordReader.getLabels() returns null, no output labels will be produced
|
||||
*
|
||||
* @param recordReader Record reader to use as the source of data
|
||||
* @param batchSize Minibatch size, for each call of .next()
|
||||
*/
|
||||
public RecordReaderDataSetIterator(RecordReader recordReader, int batchSize) {
|
||||
this(recordReader, new SelfWritableConverter(), batchSize, -1, -1,
|
||||
recordReader.getLabels() == null ? -1 : recordReader.getLabels().size(), -1, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Main constructor for classification. This will convert the input class index (at position labelIndex, with integer
|
||||
* values 0 to numPossibleLabels-1 inclusive) to the appropriate one-hot output/labels representation.
|
||||
*
|
||||
* @param recordReader RecordReader: provides the source of the data
|
||||
* @param batchSize Batch size (number of examples) for the output DataSet objects
|
||||
* @param labelIndex Index of the label Writable (usually an IntWritable), as obtained by recordReader.next()
|
||||
* @param numPossibleLabels Number of classes (possible labels) for classification
|
||||
*/
|
||||
public RecordReaderDataSetIterator(RecordReader recordReader, int batchSize, int labelIndex,
|
||||
int numPossibleLabels) {
|
||||
this(recordReader, new SelfWritableConverter(), batchSize, labelIndex, labelIndex, numPossibleLabels, -1, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor for classification, where the maximum number of returned batches is limited to the specified value
|
||||
*
|
||||
* @param recordReader the recordreader to use
|
||||
* @param labelIndex the index/column of the label (for classification)
|
||||
* @param numPossibleLabels the number of possible labels for classification. Not used if regression == true
|
||||
* @param maxNumBatches The maximum number of batches to return between resets. Set to -1 to return all available data
|
||||
*/
|
||||
public RecordReaderDataSetIterator(RecordReader recordReader, int batchSize, int labelIndex, int numPossibleLabels,
|
||||
int maxNumBatches) {
|
||||
this(recordReader, new SelfWritableConverter(), batchSize, labelIndex, labelIndex, numPossibleLabels, maxNumBatches, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Main constructor for multi-label regression (i.e., regression with multiple outputs). Can also be used for single
|
||||
* output regression with labelIndexFrom == labelIndexTo
|
||||
*
|
||||
* @param recordReader RecordReader to get data from
|
||||
* @param labelIndexFrom Index of the first regression target
|
||||
* @param labelIndexTo Index of the last regression target, inclusive
|
||||
* @param batchSize Minibatch size
|
||||
* @param regression Require regression = true. Mainly included to avoid clashing with other constructors previously defined :/
|
||||
*/
|
||||
public RecordReaderDataSetIterator(RecordReader recordReader, int batchSize, int labelIndexFrom, int labelIndexTo,
|
||||
boolean regression) {
|
||||
this(recordReader, new SelfWritableConverter(), batchSize, labelIndexFrom, labelIndexTo, -1, -1, regression);
|
||||
if (!regression) {
|
||||
throw new IllegalArgumentException("This constructor is only for creating regression iterators. " +
|
||||
"If you're doing classification you need to use another constructor that " +
|
||||
"(implicitly) specifies numPossibleLabels");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Main constructor
|
||||
*
|
||||
* @param recordReader the recordreader to use
|
||||
* @param converter Converter. May be null.
|
||||
* @param batchSize Minibatch size - number of examples returned for each call of .next()
|
||||
* @param labelIndexFrom the index of the label (for classification), or the first index of the labels for multi-output regression
|
||||
* @param labelIndexTo only used if regression == true. The last index <i>inclusive</i> of the multi-output regression
|
||||
* @param numPossibleLabels the number of possible labels for classification. Not used if regression == true
|
||||
* @param maxNumBatches Maximum number of batches to return
|
||||
* @param regression if true: regression. If false: classification (assume labelIndexFrom is the class it belongs to)
|
||||
*/
|
||||
public RecordReaderDataSetIterator(RecordReader recordReader, WritableConverter converter, int batchSize,
|
||||
int labelIndexFrom, int labelIndexTo, int numPossibleLabels, int maxNumBatches,
|
||||
boolean regression) {
|
||||
this.recordReader = recordReader;
|
||||
this.converter = converter;
|
||||
this.batchSize = batchSize;
|
||||
this.maxNumBatches = maxNumBatches;
|
||||
this.labelIndex = labelIndexFrom;
|
||||
this.labelIndexTo = labelIndexTo;
|
||||
this.numPossibleLabels = numPossibleLabels;
|
||||
this.regression = regression;
|
||||
}
|
||||
|
||||
|
||||
protected RecordReaderDataSetIterator(Builder b){
|
||||
this.recordReader = b.recordReader;
|
||||
this.converter = b.converter;
|
||||
this.batchSize = b.batchSize;
|
||||
this.maxNumBatches = b.maxNumBatches;
|
||||
this.labelIndex = b.labelIndex;
|
||||
this.labelIndexTo = b.labelIndexTo;
|
||||
this.numPossibleLabels = b.numPossibleLabels;
|
||||
this.regression = b.regression;
|
||||
this.preProcessor = b.preProcessor;
|
||||
this.collectMetaData = b.collectMetaData;
|
||||
}
|
||||
|
||||
/**
|
||||
* When set to true: metadata for the current examples will be present in the returned DataSet.
|
||||
* Disabled by default.
|
||||
*
|
||||
* @param collectMetaData Whether to collect metadata or not
|
||||
*/
|
||||
public void setCollectMetaData(boolean collectMetaData) {
|
||||
if (underlying != null) {
|
||||
underlying.setCollectMetaData(collectMetaData);
|
||||
}
|
||||
this.collectMetaData = collectMetaData;
|
||||
}
|
||||
|
||||
private void initializeUnderlying() {
|
||||
if (underlying == null) {
|
||||
Record next = recordReader.nextRecord();
|
||||
initializeUnderlying(next);
|
||||
}
|
||||
}
|
||||
|
||||
private void initializeUnderlying(Record next) {
|
||||
int totalSize = next.getRecord().size();
|
||||
|
||||
//allow people to specify label index as -1 and infer the last possible label
|
||||
if (numPossibleLabels >= 1 && labelIndex < 0) {
|
||||
labelIndex = totalSize - 1;
|
||||
labelIndexTo = labelIndex;
|
||||
}
|
||||
|
||||
if(recordReader.resetSupported()) {
|
||||
recordReader.reset();
|
||||
} else {
|
||||
//Hack around the fact that we need the first record to initialize the underlying RRMDSI, but can't reset
|
||||
// the original reader
|
||||
recordReader = new ConcatenatingRecordReader(
|
||||
new CollectionRecordReader(Collections.singletonList(next.getRecord())),
|
||||
recordReader);
|
||||
}
|
||||
|
||||
RecordReaderMultiDataSetIterator.Builder builder = new RecordReaderMultiDataSetIterator.Builder(batchSize);
|
||||
if (recordReader instanceof SequenceRecordReader) {
|
||||
builder.addSequenceReader(READER_KEY, (SequenceRecordReader) recordReader);
|
||||
} else {
|
||||
builder.addReader(READER_KEY, recordReader);
|
||||
}
|
||||
|
||||
if (regression) {
|
||||
builder.addOutput(READER_KEY, labelIndex, labelIndexTo);
|
||||
} else if (numPossibleLabels >= 1) {
|
||||
builder.addOutputOneHot(READER_KEY, labelIndex, numPossibleLabels);
|
||||
}
|
||||
|
||||
//Inputs: assume to be all the other writables
|
||||
//In general: can't assume label indices are all at the start or end (event though 99% of the time they are)
|
||||
//If they are: easy. If not: use 2 inputs in the underlying as a workaround, and concat them
|
||||
|
||||
if (labelIndex >= 0 && (labelIndex == 0 || labelIndexTo == totalSize - 1)) {
|
||||
//Labels are first or last -> one input in underlying
|
||||
int inputFrom;
|
||||
int inputTo;
|
||||
if (labelIndex < 0) {
|
||||
//No label
|
||||
inputFrom = 0;
|
||||
inputTo = totalSize - 1;
|
||||
} else if (labelIndex == 0) {
|
||||
inputFrom = labelIndexTo + 1;
|
||||
inputTo = totalSize - 1;
|
||||
} else {
|
||||
inputFrom = 0;
|
||||
inputTo = labelIndex - 1;
|
||||
}
|
||||
|
||||
builder.addInput(READER_KEY, inputFrom, inputTo);
|
||||
|
||||
underlyingIsDisjoint = false;
|
||||
} else if (labelIndex >= 0) {
|
||||
Preconditions.checkState(labelIndex < next.getRecord().size(),
|
||||
"Invalid label (from) index: index must be in range 0 to first record size of (0 to %s inclusive), got %s", next.getRecord().size()-1, labelIndex);
|
||||
Preconditions.checkState(labelIndexTo < next.getRecord().size(),
|
||||
"Invalid label (to) index: index must be in range 0 to first record size of (0 to %s inclusive), got %s", next.getRecord().size()-1, labelIndexTo);
|
||||
|
||||
|
||||
//Multiple inputs
|
||||
int firstFrom = 0;
|
||||
int firstTo = labelIndex - 1;
|
||||
int secondFrom = labelIndexTo + 1;
|
||||
int secondTo = totalSize - 1;
|
||||
|
||||
builder.addInput(READER_KEY, firstFrom, firstTo);
|
||||
builder.addInput(READER_KEY, secondFrom, secondTo);
|
||||
|
||||
underlyingIsDisjoint = true;
|
||||
} else {
|
||||
//No labels - only features
|
||||
builder.addInput(READER_KEY);
|
||||
underlyingIsDisjoint = false;
|
||||
}
|
||||
|
||||
|
||||
underlying = builder.build();
|
||||
|
||||
if (collectMetaData) {
|
||||
underlying.setCollectMetaData(true);
|
||||
}
|
||||
}
|
||||
|
||||
private DataSet mdsToDataSet(MultiDataSet mds) {
|
||||
INDArray f;
|
||||
INDArray fm;
|
||||
if (underlyingIsDisjoint) {
|
||||
//Rare case: 2 input arrays -> concat
|
||||
INDArray f1 = getOrNull(mds.getFeatures(), 0);
|
||||
INDArray f2 = getOrNull(mds.getFeatures(), 1);
|
||||
fm = getOrNull(mds.getFeaturesMaskArrays(), 0); //Per-example masking only on the input -> same for both
|
||||
|
||||
//Can assume 2d features here
|
||||
f = Nd4j.hstack(f1, f2);
|
||||
} else {
|
||||
//Standard case
|
||||
f = getOrNull(mds.getFeatures(), 0);
|
||||
fm = getOrNull(mds.getFeaturesMaskArrays(), 0);
|
||||
}
|
||||
|
||||
INDArray l = getOrNull(mds.getLabels(), 0);
|
||||
INDArray lm = getOrNull(mds.getLabelsMaskArrays(), 0);
|
||||
|
||||
DataSet ds = new DataSet(f, l, fm, lm);
|
||||
|
||||
if (collectMetaData) {
|
||||
List<Serializable> temp = mds.getExampleMetaData();
|
||||
List<Serializable> temp2 = new ArrayList<>(temp.size());
|
||||
for (Serializable s : temp) {
|
||||
RecordMetaDataComposableMap m = (RecordMetaDataComposableMap) s;
|
||||
temp2.add(m.getMeta().get(READER_KEY));
|
||||
}
|
||||
ds.setExampleMetaData(temp2);
|
||||
}
|
||||
|
||||
//Edge case, for backward compatibility:
|
||||
//If labelIdx == -1 && numPossibleLabels == -1 -> no labels -> set labels array to features array
|
||||
if (labelIndex == -1 && numPossibleLabels == -1 && ds.getLabels() == null) {
|
||||
ds.setLabels(ds.getFeatures());
|
||||
}
|
||||
|
||||
if (preProcessor != null) {
|
||||
preProcessor.preProcess(ds);
|
||||
}
|
||||
|
||||
return ds;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public DataSet next(int num) {
|
||||
if (useCurrent) {
|
||||
useCurrent = false;
|
||||
if (preProcessor != null)
|
||||
preProcessor.preProcess(last);
|
||||
return last;
|
||||
}
|
||||
|
||||
if (underlying == null) {
|
||||
initializeUnderlying();
|
||||
}
|
||||
|
||||
|
||||
batchNum++;
|
||||
return mdsToDataSet(underlying.next(num));
|
||||
}
|
||||
|
||||
//Package private
|
||||
static INDArray getOrNull(INDArray[] arr, int idx) {
|
||||
if (arr == null || arr.length == 0) {
|
||||
return null;
|
||||
}
|
||||
return arr[idx];
|
||||
}
|
||||
|
||||
@Override
|
||||
public int inputColumns() {
|
||||
if (last == null) {
|
||||
DataSet next = next();
|
||||
last = next;
|
||||
useCurrent = true;
|
||||
return next.numInputs();
|
||||
} else
|
||||
return last.numInputs();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int totalOutcomes() {
|
||||
if (last == null) {
|
||||
DataSet next = next();
|
||||
last = next;
|
||||
useCurrent = true;
|
||||
return next.numOutcomes();
|
||||
} else
|
||||
return last.numOutcomes();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean resetSupported() {
|
||||
if(underlying == null){
|
||||
initializeUnderlying();
|
||||
}
|
||||
return underlying.resetSupported();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean asyncSupported() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reset() {
|
||||
batchNum = 0;
|
||||
if (underlying != null) {
|
||||
underlying.reset();
|
||||
}
|
||||
|
||||
last = null;
|
||||
useCurrent = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int batch() {
|
||||
return batchSize;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setPreProcessor(org.nd4j.linalg.dataset.api.DataSetPreProcessor preProcessor) {
|
||||
this.preProcessor = preProcessor;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return (((sequenceIter != null && sequenceIter.hasNext()) || recordReader.hasNext())
|
||||
&& (maxNumBatches < 0 || batchNum < maxNumBatches));
|
||||
}
|
||||
|
||||
@Override
|
||||
public DataSet next() {
|
||||
return next(batchSize);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getLabels() {
|
||||
return recordReader.getLabels();
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a single example to a DataSet, using the provided RecordMetaData.
|
||||
* Note that it is more efficient to load multiple instances at once, using {@link #loadFromMetaData(List)}
|
||||
*
|
||||
* @param recordMetaData RecordMetaData to load from. Should have been produced by the given record reader
|
||||
* @return DataSet with the specified example
|
||||
* @throws IOException If an error occurs during loading of the data
|
||||
*/
|
||||
public DataSet loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
|
||||
return loadFromMetaData(Collections.singletonList(recordMetaData));
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a multiple examples to a DataSet, using the provided RecordMetaData instances.
|
||||
*
|
||||
* @param list List of RecordMetaData instances to load from. Should have been produced by the record reader provided
|
||||
* to the RecordReaderDataSetIterator constructor
|
||||
* @return DataSet with the specified examples
|
||||
* @throws IOException If an error occurs during loading of the data
|
||||
*/
|
||||
public DataSet loadFromMetaData(List<RecordMetaData> list) throws IOException {
|
||||
if (underlying == null) {
|
||||
Record r = recordReader.loadFromMetaData(list.get(0));
|
||||
initializeUnderlying(r);
|
||||
}
|
||||
|
||||
//Convert back to composable:
|
||||
List<RecordMetaData> l = new ArrayList<>(list.size());
|
||||
for (RecordMetaData m : list) {
|
||||
l.add(new RecordMetaDataComposableMap(Collections.singletonMap(READER_KEY, m)));
|
||||
}
|
||||
MultiDataSet m = underlying.loadFromMetaData(l);
|
||||
|
||||
return mdsToDataSet(m);
|
||||
}
|
||||
|
||||
/**
|
||||
* Builder class for RecordReaderDataSetIterator
|
||||
*/
|
||||
public static class Builder {
|
||||
|
||||
protected RecordReader recordReader;
|
||||
protected WritableConverter converter;
|
||||
protected int batchSize;
|
||||
protected int maxNumBatches = -1;
|
||||
protected int labelIndex = -1;
|
||||
protected int labelIndexTo = -1;
|
||||
protected int numPossibleLabels = -1;
|
||||
protected boolean regression = false;
|
||||
protected DataSetPreProcessor preProcessor;
|
||||
private boolean collectMetaData = false;
|
||||
|
||||
private boolean clOrRegCalled = false;
|
||||
|
||||
/**
|
||||
*
|
||||
* @param rr Underlying record reader to source data from
|
||||
* @param batchSize Batch size to use
|
||||
*/
|
||||
public Builder(@NonNull RecordReader rr, int batchSize){
|
||||
this.recordReader = rr;
|
||||
this.batchSize = batchSize;
|
||||
}
|
||||
|
||||
public Builder writableConverter(WritableConverter converter){
|
||||
this.converter = converter;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Optional argument, usually not used. If set, can be used to limit the maximum number of minibatches that
|
||||
* will be returned (between resets). If not set, will always return as many minibatches as there is data
|
||||
* available.
|
||||
*
|
||||
* @param maxNumBatches Maximum number of minibatches per epoch / reset
|
||||
*/
|
||||
public Builder maxNumBatches(int maxNumBatches){
|
||||
this.maxNumBatches = maxNumBatches;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Use this for single output regression (i.e., 1 output/regression target)
|
||||
*
|
||||
* @param labelIndex Column index that contains the regression target (indexes start at 0)
|
||||
*/
|
||||
public Builder regression(int labelIndex){
|
||||
return regression(labelIndex, labelIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* Use this for multiple output regression (1 or more output/regression targets). Note that all regression
|
||||
* targets must be contiguous (i.e., positions x to y, without gaps)
|
||||
*
|
||||
* @param labelIndexFrom Column index of the first regression target (indexes start at 0)
|
||||
* @param labelIndexTo Column index of the last regression target (inclusive)
|
||||
*/
|
||||
public Builder regression(int labelIndexFrom, int labelIndexTo){
|
||||
this.labelIndex = labelIndexFrom;
|
||||
this.labelIndexTo = labelIndexTo;
|
||||
this.regression = true;
|
||||
clOrRegCalled = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Use this for classification
|
||||
*
|
||||
* @param labelIndex Index that contains the label index. Column (indexes start from 0) be an integer value,
|
||||
* and contain values 0 to numClasses-1
|
||||
* @param numClasses Number of label classes (i.e., number of categories/classes in the dataset)
|
||||
*/
|
||||
public Builder classification(int labelIndex, int numClasses){
|
||||
this.labelIndex = labelIndex;
|
||||
this.labelIndexTo = labelIndex;
|
||||
this.numPossibleLabels = numClasses;
|
||||
this.regression = false;
|
||||
clOrRegCalled = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Optional arg. Allows the preprocessor to be set
|
||||
* @param preProcessor Preprocessor to use
|
||||
*/
|
||||
public Builder preProcessor(DataSetPreProcessor preProcessor){
|
||||
this.preProcessor = preProcessor;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* When set to true: metadata for the current examples will be present in the returned DataSet.
|
||||
* Disabled by default.
|
||||
*
|
||||
* @param collectMetaData Whether metadata should be collected or not
|
||||
*/
|
||||
public Builder collectMetaData(boolean collectMetaData){
|
||||
this.collectMetaData = collectMetaData;
|
||||
return this;
|
||||
}
|
||||
|
||||
public RecordReaderDataSetIterator build(){
|
||||
return new RecordReaderDataSetIterator(this);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+1022
File diff suppressed because it is too large
Load Diff
+480
@@ -0,0 +1,480 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.deeplearning4j.datasets.datavec;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.datavec.api.records.SequenceRecord;
|
||||
import org.datavec.api.records.metadata.RecordMetaData;
|
||||
import org.datavec.api.records.metadata.RecordMetaDataComposable;
|
||||
import org.datavec.api.records.metadata.RecordMetaDataComposableMap;
|
||||
import org.datavec.api.records.reader.SequenceRecordReader;
|
||||
import org.deeplearning4j.datasets.datavec.exception.ZeroLengthSequenceException;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.dataset.DataSet;
|
||||
import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
|
||||
import org.nd4j.linalg.dataset.api.MultiDataSet;
|
||||
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
|
||||
import org.nd4j.linalg.factory.Nd4j;
|
||||
import org.nd4j.linalg.indexing.INDArrayIndex;
|
||||
import org.nd4j.linalg.indexing.NDArrayIndex;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Serializable;
|
||||
import java.util.*;
|
||||
|
||||
public class SequenceRecordReaderDataSetIterator implements DataSetIterator {
|
||||
/**Alignment mode for dealing with input/labels of differing lengths (for example, one-to-many and many-to-one type situations).
|
||||
* For example, might have 10 time steps total but only one label at end for sequence classification.<br>
|
||||
* Currently supported modes:<br>
|
||||
* <b>EQUAL_LENGTH</b>: Default. Assume that label and input time series are of equal length, and all examples are of
|
||||
* the same length<br>
|
||||
* <b>ALIGN_START</b>: Align the label/input time series at the first time step, and zero pad either the labels or
|
||||
* the input at the end<br>
|
||||
* <b>ALIGN_END</b>: Align the label/input at the last time step, zero padding either the input or the labels as required<br>
|
||||
*
|
||||
* Note 1: When the time series for each example are of different lengths, the shorter time series will be padded to
|
||||
* the length of the longest time series.<br>
|
||||
* Note 2: When ALIGN_START or ALIGN_END are used, the DataSet masking functionality is used. Thus, the returned DataSets
|
||||
* will have the input and mask arrays set. These mask arrays identify whether an input/label is actually present,
|
||||
* or whether the value is merely masked.<br>
|
||||
*/
|
||||
public enum AlignmentMode {
|
||||
EQUAL_LENGTH, ALIGN_START, ALIGN_END
|
||||
}
|
||||
|
||||
private static final String READER_KEY = "reader";
|
||||
private static final String READER_KEY_LABEL = "reader_labels";
|
||||
|
||||
private SequenceRecordReader recordReader;
|
||||
private SequenceRecordReader labelsReader;
|
||||
private int miniBatchSize = 10;
|
||||
private final boolean regression;
|
||||
private int labelIndex = -1;
|
||||
private final int numPossibleLabels;
|
||||
private int cursor = 0;
|
||||
private int inputColumns = -1;
|
||||
private int totalOutcomes = -1;
|
||||
private boolean useStored = false;
|
||||
private DataSet stored = null;
|
||||
@Getter
|
||||
private DataSetPreProcessor preProcessor;
|
||||
private AlignmentMode alignmentMode;
|
||||
|
||||
private final boolean singleSequenceReaderMode;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
private boolean collectMetaData = false;
|
||||
|
||||
private RecordReaderMultiDataSetIterator underlying;
|
||||
private boolean underlyingIsDisjoint;
|
||||
|
||||
/**
|
||||
* Constructor where features and labels come from different RecordReaders (for example, different files),
|
||||
* and labels are for classification.
|
||||
*
|
||||
* @param featuresReader SequenceRecordReader for the features
|
||||
* @param labels Labels: assume single value per time step, where values are integers in the range 0 to numPossibleLables-1
|
||||
* @param miniBatchSize Minibatch size for each call of next()
|
||||
* @param numPossibleLabels Number of classes for the labels
|
||||
*/
|
||||
public SequenceRecordReaderDataSetIterator(SequenceRecordReader featuresReader, SequenceRecordReader labels,
|
||||
int miniBatchSize, int numPossibleLabels) {
|
||||
this(featuresReader, labels, miniBatchSize, numPossibleLabels, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor where features and labels come from different RecordReaders (for example, different files)
|
||||
*/
|
||||
public SequenceRecordReaderDataSetIterator(SequenceRecordReader featuresReader, SequenceRecordReader labels,
|
||||
int miniBatchSize, int numPossibleLabels, boolean regression) {
|
||||
this(featuresReader, labels, miniBatchSize, numPossibleLabels, regression, AlignmentMode.EQUAL_LENGTH);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor where features and labels come from different RecordReaders (for example, different files)
|
||||
*/
|
||||
public SequenceRecordReaderDataSetIterator(SequenceRecordReader featuresReader, SequenceRecordReader labels,
|
||||
int miniBatchSize, int numPossibleLabels, boolean regression, AlignmentMode alignmentMode) {
|
||||
this.recordReader = featuresReader;
|
||||
this.labelsReader = labels;
|
||||
this.miniBatchSize = miniBatchSize;
|
||||
this.numPossibleLabels = numPossibleLabels;
|
||||
this.regression = regression;
|
||||
this.alignmentMode = alignmentMode;
|
||||
this.singleSequenceReaderMode = false;
|
||||
}
|
||||
|
||||
/** Constructor where features and labels come from the SAME RecordReader (i.e., target/label is a column in the
|
||||
* same data as the features). Defaults to regression = false - i.e., for classification
|
||||
* @param reader SequenceRecordReader with data
|
||||
* @param miniBatchSize size of each minibatch
|
||||
* @param numPossibleLabels number of labels/classes for classification
|
||||
* @param labelIndex index in input of the label index. If in regression mode and numPossibleLabels > 1, labelIndex denotes the
|
||||
* first index for labels. Everything before that index will be treated as input(s) and
|
||||
* everything from that index (inclusive) to the end will be treated as output(s)
|
||||
*/
|
||||
public SequenceRecordReaderDataSetIterator(SequenceRecordReader reader, int miniBatchSize, int numPossibleLabels,
|
||||
int labelIndex) {
|
||||
this(reader, miniBatchSize, numPossibleLabels, labelIndex, false);
|
||||
}
|
||||
|
||||
/** Constructor where features and labels come from the SAME RecordReader (i.e., target/label is a column in the
|
||||
* same data as the features)
|
||||
* @param reader SequenceRecordReader with data
|
||||
* @param miniBatchSize size of each minibatch
|
||||
* @param numPossibleLabels number of labels/classes for classification
|
||||
* @param labelIndex index in input of the label index. If in regression mode and numPossibleLabels > 1, labelIndex denotes the
|
||||
* first index for labels. Everything before that index will be treated as input(s) and
|
||||
* everything from that index (inclusive) to the end will be treated as output(s)
|
||||
* @param regression Whether output is for regression or classification
|
||||
*/
|
||||
public SequenceRecordReaderDataSetIterator(SequenceRecordReader reader, int miniBatchSize, int numPossibleLabels,
|
||||
int labelIndex, boolean regression) {
|
||||
this.recordReader = reader;
|
||||
this.labelsReader = null;
|
||||
this.miniBatchSize = miniBatchSize;
|
||||
this.regression = regression;
|
||||
this.labelIndex = labelIndex;
|
||||
this.numPossibleLabels = numPossibleLabels;
|
||||
this.singleSequenceReaderMode = true;
|
||||
}
|
||||
|
||||
private void initializeUnderlyingFromReader() {
|
||||
initializeUnderlying(recordReader.nextSequence());
|
||||
underlying.reset();
|
||||
}
|
||||
|
||||
private void initializeUnderlying(SequenceRecord nextF) {
|
||||
if (nextF.getSequenceRecord().isEmpty()) {
|
||||
throw new ZeroLengthSequenceException();
|
||||
}
|
||||
int totalSizeF = nextF.getSequenceRecord().get(0).size();
|
||||
|
||||
//allow people to specify label index as -1 and infer the last possible label
|
||||
if (singleSequenceReaderMode && numPossibleLabels >= 1 && labelIndex < 0) {
|
||||
labelIndex = totalSizeF - 1;
|
||||
} else if (!singleSequenceReaderMode && numPossibleLabels >= 1 && labelIndex < 0) {
|
||||
labelIndex = 0;
|
||||
}
|
||||
|
||||
recordReader.reset();
|
||||
|
||||
//Add readers
|
||||
RecordReaderMultiDataSetIterator.Builder builder = new RecordReaderMultiDataSetIterator.Builder(miniBatchSize);
|
||||
builder.addSequenceReader(READER_KEY, recordReader);
|
||||
if (labelsReader != null) {
|
||||
builder.addSequenceReader(READER_KEY_LABEL, labelsReader);
|
||||
}
|
||||
|
||||
|
||||
//Add outputs
|
||||
if (singleSequenceReaderMode) {
|
||||
|
||||
if (labelIndex < 0 && numPossibleLabels < 0) {
|
||||
//No labels - all values -> features array
|
||||
builder.addInput(READER_KEY);
|
||||
} else if (labelIndex == 0 || labelIndex == totalSizeF - 1) { //Features: subset of columns
|
||||
//Labels are first or last -> one input in underlying
|
||||
int inputFrom;
|
||||
int inputTo;
|
||||
if (labelIndex < 0) {
|
||||
//No label
|
||||
inputFrom = 0;
|
||||
inputTo = totalSizeF - 1;
|
||||
} else if (labelIndex == 0) {
|
||||
inputFrom = 1;
|
||||
inputTo = totalSizeF - 1;
|
||||
} else {
|
||||
inputFrom = 0;
|
||||
inputTo = labelIndex - 1;
|
||||
}
|
||||
|
||||
builder.addInput(READER_KEY, inputFrom, inputTo);
|
||||
|
||||
underlyingIsDisjoint = false;
|
||||
} else if (regression && numPossibleLabels > 1){
|
||||
//Multiple inputs and multiple outputs
|
||||
int inputFrom = 0;
|
||||
int inputTo = labelIndex - 1;
|
||||
int outputFrom = labelIndex;
|
||||
int outputTo = totalSizeF - 1;
|
||||
|
||||
builder.addInput(READER_KEY, inputFrom, inputTo);
|
||||
builder.addOutput(READER_KEY, outputFrom, outputTo);
|
||||
|
||||
underlyingIsDisjoint = false;
|
||||
} else {
|
||||
//Multiple inputs (disjoint features case)
|
||||
int firstFrom = 0;
|
||||
int firstTo = labelIndex - 1;
|
||||
int secondFrom = labelIndex + 1;
|
||||
int secondTo = totalSizeF - 1;
|
||||
|
||||
builder.addInput(READER_KEY, firstFrom, firstTo);
|
||||
builder.addInput(READER_KEY, secondFrom, secondTo);
|
||||
|
||||
underlyingIsDisjoint = true;
|
||||
}
|
||||
|
||||
if(!(labelIndex < 0 && numPossibleLabels < 0)) {
|
||||
if (regression && numPossibleLabels <= 1) {
|
||||
//Multiple output regression already handled
|
||||
builder.addOutput(READER_KEY, labelIndex, labelIndex);
|
||||
} else if (!regression) {
|
||||
builder.addOutputOneHot(READER_KEY, labelIndex, numPossibleLabels);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
//Features: entire reader
|
||||
builder.addInput(READER_KEY);
|
||||
underlyingIsDisjoint = false;
|
||||
|
||||
if (regression) {
|
||||
builder.addOutput(READER_KEY_LABEL);
|
||||
} else {
|
||||
builder.addOutputOneHot(READER_KEY_LABEL, 0, numPossibleLabels);
|
||||
}
|
||||
}
|
||||
|
||||
if (alignmentMode != null) {
|
||||
switch (alignmentMode) {
|
||||
case EQUAL_LENGTH:
|
||||
builder.sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.EQUAL_LENGTH);
|
||||
break;
|
||||
case ALIGN_START:
|
||||
builder.sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.ALIGN_START);
|
||||
break;
|
||||
case ALIGN_END:
|
||||
builder.sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.ALIGN_END);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
underlying = builder.build();
|
||||
|
||||
if (collectMetaData) {
|
||||
underlying.setCollectMetaData(true);
|
||||
}
|
||||
}
|
||||
|
||||
private DataSet mdsToDataSet(MultiDataSet mds) {
|
||||
INDArray f;
|
||||
INDArray fm;
|
||||
if (underlyingIsDisjoint) {
|
||||
//Rare case: 2 input arrays -> concat
|
||||
INDArray f1 = RecordReaderDataSetIterator.getOrNull(mds.getFeatures(), 0);
|
||||
INDArray f2 = RecordReaderDataSetIterator.getOrNull(mds.getFeatures(), 1);
|
||||
fm = RecordReaderDataSetIterator.getOrNull(mds.getFeaturesMaskArrays(), 0); //Per-example masking only on the input -> same for both
|
||||
|
||||
//Can assume 3d features here
|
||||
f = Nd4j.createUninitialized(new long[] {f1.size(0), f1.size(1) + f2.size(1), f1.size(2)});
|
||||
f.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.interval(0, f1.size(1)), NDArrayIndex.all()},
|
||||
f1);
|
||||
f.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.interval(f1.size(1), f1.size(1) + f2.size(1)),
|
||||
NDArrayIndex.all()}, f2);
|
||||
} else {
|
||||
//Standard case
|
||||
f = RecordReaderDataSetIterator.getOrNull(mds.getFeatures(), 0);
|
||||
fm = RecordReaderDataSetIterator.getOrNull(mds.getFeaturesMaskArrays(), 0);
|
||||
}
|
||||
|
||||
INDArray l = RecordReaderDataSetIterator.getOrNull(mds.getLabels(), 0);
|
||||
INDArray lm = RecordReaderDataSetIterator.getOrNull(mds.getLabelsMaskArrays(), 0);
|
||||
|
||||
DataSet ds = new DataSet(f, l, fm, lm);
|
||||
|
||||
if (collectMetaData) {
|
||||
List<Serializable> temp = mds.getExampleMetaData();
|
||||
List<Serializable> temp2 = new ArrayList<>(temp.size());
|
||||
for (Serializable s : temp) {
|
||||
RecordMetaDataComposableMap m = (RecordMetaDataComposableMap) s;
|
||||
if (singleSequenceReaderMode) {
|
||||
temp2.add(m.getMeta().get(READER_KEY));
|
||||
} else {
|
||||
RecordMetaDataComposable c = new RecordMetaDataComposable(m.getMeta().get(READER_KEY),
|
||||
m.getMeta().get(READER_KEY_LABEL));
|
||||
temp2.add(c);
|
||||
}
|
||||
}
|
||||
ds.setExampleMetaData(temp2);
|
||||
}
|
||||
|
||||
if (preProcessor != null) {
|
||||
preProcessor.preProcess(ds);
|
||||
}
|
||||
|
||||
return ds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
if (underlying == null) {
|
||||
initializeUnderlyingFromReader();
|
||||
}
|
||||
return underlying.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public DataSet next() {
|
||||
return next(miniBatchSize);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public DataSet next(int num) {
|
||||
if (useStored) {
|
||||
useStored = false;
|
||||
DataSet temp = stored;
|
||||
stored = null;
|
||||
if (preProcessor != null)
|
||||
preProcessor.preProcess(temp);
|
||||
return temp;
|
||||
}
|
||||
if (!hasNext())
|
||||
throw new NoSuchElementException();
|
||||
|
||||
if (underlying == null) {
|
||||
initializeUnderlyingFromReader();
|
||||
}
|
||||
|
||||
MultiDataSet mds = underlying.next(num);
|
||||
DataSet ds = mdsToDataSet(mds);
|
||||
|
||||
if (totalOutcomes == -1) {
|
||||
inputColumns = (int) ds.getFeatures().size(1);
|
||||
totalOutcomes = ds.getLabels() == null ? -1 : (int) ds.getLabels().size(1);
|
||||
}
|
||||
|
||||
return ds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int inputColumns() {
|
||||
if (inputColumns != -1)
|
||||
return inputColumns;
|
||||
preLoad();
|
||||
return inputColumns;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int totalOutcomes() {
|
||||
if (totalOutcomes != -1)
|
||||
return totalOutcomes;
|
||||
preLoad();
|
||||
return totalOutcomes;
|
||||
}
|
||||
|
||||
private void preLoad() {
|
||||
stored = next();
|
||||
useStored = true;
|
||||
|
||||
inputColumns = (int) stored.getFeatures().size(1);
|
||||
totalOutcomes = (int) stored.getLabels().size(1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean resetSupported() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean asyncSupported() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reset() {
|
||||
if (underlying != null)
|
||||
underlying.reset();
|
||||
|
||||
cursor = 0;
|
||||
stored = null;
|
||||
useStored = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int batch() {
|
||||
return miniBatchSize;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setPreProcessor(DataSetPreProcessor preProcessor) {
|
||||
this.preProcessor = preProcessor;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getLabels() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
throw new UnsupportedOperationException("Remove not supported for this iterator");
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a single sequence example to a DataSet, using the provided RecordMetaData.
|
||||
* Note that it is more efficient to load multiple instances at once, using {@link #loadFromMetaData(List)}
|
||||
*
|
||||
* @param recordMetaData RecordMetaData to load from. Should have been produced by the given record reader
|
||||
* @return DataSet with the specified example
|
||||
* @throws IOException If an error occurs during loading of the data
|
||||
*/
|
||||
public DataSet loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
|
||||
return loadFromMetaData(Collections.singletonList(recordMetaData));
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a multiple sequence examples to a DataSet, using the provided RecordMetaData instances.
|
||||
*
|
||||
* @param list List of RecordMetaData instances to load from. Should have been produced by the record reader provided
|
||||
* to the SequenceRecordReaderDataSetIterator constructor
|
||||
* @return DataSet with the specified examples
|
||||
* @throws IOException If an error occurs during loading of the data
|
||||
*/
|
||||
public DataSet loadFromMetaData(List<RecordMetaData> list) throws IOException {
|
||||
if (underlying == null) {
|
||||
SequenceRecord r = recordReader.loadSequenceFromMetaData(list.get(0));
|
||||
initializeUnderlying(r);
|
||||
}
|
||||
|
||||
//Two cases: single vs. multiple reader...
|
||||
List<RecordMetaData> l = new ArrayList<>(list.size());
|
||||
if (singleSequenceReaderMode) {
|
||||
for (RecordMetaData m : list) {
|
||||
l.add(new RecordMetaDataComposableMap(Collections.singletonMap(READER_KEY, m)));
|
||||
}
|
||||
} else {
|
||||
for (RecordMetaData m : list) {
|
||||
RecordMetaDataComposable rmdc = (RecordMetaDataComposable) m;
|
||||
Map<String, RecordMetaData> map = new HashMap<>(2);
|
||||
map.put(READER_KEY, rmdc.getMeta()[0]);
|
||||
map.put(READER_KEY_LABEL, rmdc.getMeta()[1]);
|
||||
l.add(new RecordMetaDataComposableMap(map));
|
||||
}
|
||||
}
|
||||
|
||||
return mdsToDataSet(underlying.loadFromMetaData(l));
|
||||
}
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.deeplearning4j.datasets.datavec.exception;
|
||||
|
||||
public class ZeroLengthSequenceException extends RuntimeException {
|
||||
public ZeroLengthSequenceException() {
|
||||
this("");
|
||||
}
|
||||
|
||||
public ZeroLengthSequenceException(String type) {
|
||||
super(String.format("Encountered zero-length %ssequence", type.equals("") ? "" : type + " "));
|
||||
}
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
open module deeplearning4j.datavec.iterators {
|
||||
requires nd4j.common;
|
||||
requires org.apache.commons.lang3;
|
||||
requires slf4j.api;
|
||||
requires datavec.api;
|
||||
requires nd4j.api;
|
||||
exports org.deeplearning4j.datasets.datavec;
|
||||
exports org.deeplearning4j.datasets.datavec.exception;
|
||||
}
|
||||
Reference in New Issue
Block a user