chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:47:05 +08:00
commit 4f3b7da785
7394 changed files with 2005594 additions and 0 deletions
@@ -0,0 +1,34 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.data;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.nd4j.linalg.api.ndarray.INDArray;
@AllArgsConstructor
@Data
public class Image {
private INDArray image;
private int origC;
private int origH;
private int origW;
}
@@ -0,0 +1,130 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.data;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.FrameConverter;
import org.datavec.api.writable.Writable;
import org.datavec.api.writable.WritableFactory;
import org.datavec.api.writable.WritableType;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.Buffer;
public class ImageWritable implements Writable {
static {
WritableFactory.getInstance().registerWritableType(WritableType.Image.typeIdx(), ImageWritable.class);
}
protected Frame frame;
public ImageWritable() {
//No-arg cosntructor for reflection-based creation of ImageWritable objects
}
public ImageWritable(Frame frame) {
this.frame = frame;
}
public Frame getFrame() {
return frame;
}
public void setFrame(Frame frame) {
this.frame = frame;
}
public int getWidth() {
return frame.imageWidth;
}
public int getHeight() {
return frame.imageHeight;
}
public int getDepth() {
return frame.imageDepth;
}
@Override
public void write(DataOutput out) throws IOException {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public void readFields(DataInput in) throws IOException {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public void writeType(DataOutput out) throws IOException {
out.writeShort(WritableType.Image.typeIdx());
}
@Override
public double toDouble() {
throw new UnsupportedOperationException();
}
@Override
public float toFloat() {
throw new UnsupportedOperationException();
}
@Override
public int toInt() {
throw new UnsupportedOperationException();
}
@Override
public long toLong() {
throw new UnsupportedOperationException();
}
@Override
public WritableType getType() {
return WritableType.Image;
}
@Override
public boolean equals(Object obj) {
if (obj instanceof ImageWritable) {
Frame f2 = ((ImageWritable) obj).getFrame();
Buffer[] b1 = this.frame.image;
Buffer[] b2 = f2.image;
if (b1.length != b2.length)
return false;
for (int i = 0; i < b1.length; i++) {
if (!b1[i].equals(b2[i]))
return false;
}
return true;
} else {
return false;
}
}
}
@@ -0,0 +1,47 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.format;
import org.datavec.api.conf.Configuration;
import org.datavec.api.formats.input.BaseInputFormat;
import org.datavec.api.records.reader.RecordReader;
import org.datavec.api.split.InputSplit;
import org.datavec.image.recordreader.ImageRecordReader;
import java.io.IOException;
/**
* @author Adam Gibson
*/
public class ImageInputFormat extends BaseInputFormat {
@Override
public RecordReader createReader(InputSplit split, Configuration conf) throws IOException, InterruptedException {
RecordReader reader = new ImageRecordReader();
reader.initialize(conf, split);
return reader;
}
@Override
public RecordReader createReader(InputSplit split) throws IOException, InterruptedException {
return createReader(split, new Configuration());
}
}
@@ -0,0 +1,94 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.loader;
import android.graphics.Bitmap;
import org.bytedeco.javacv.AndroidFrameConverter;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.datavec.image.transform.ImageTransform;
import org.nd4j.linalg.api.ndarray.INDArray;
import java.io.IOException;
public class AndroidNativeImageLoader extends NativeImageLoader {
AndroidFrameConverter converter2 = new AndroidFrameConverter();
public AndroidNativeImageLoader() {}
public AndroidNativeImageLoader(int height, int width) {
super(height, width);
}
public AndroidNativeImageLoader(int height, int width, int channels) {
super(height, width, channels);
}
public AndroidNativeImageLoader(int height, int width, int channels, boolean centerCropIfNeeded) {
super(height, width, channels, centerCropIfNeeded);
}
public AndroidNativeImageLoader(int height, int width, int channels, ImageTransform imageTransform) {
super(height, width, channels, imageTransform);
}
protected AndroidNativeImageLoader(NativeImageLoader other) {
super(other);
}
public INDArray asRowVector(Bitmap image) throws IOException {
return asMatrix(image).ravel();
}
public INDArray asMatrix(Bitmap image) throws IOException {
if (converter == null) {
converter = new OpenCVFrameConverter.ToMat();
}
return asMatrix(converter.convert(converter2.convert(image)));
}
@Override
public INDArray asRowVector(Object image) throws IOException {
return image instanceof Bitmap ? asRowVector((Bitmap) image) : null;
}
@Override
public INDArray asMatrix(Object image) throws IOException {
return image instanceof Bitmap ? asMatrix((Bitmap) image) : null;
}
/** Returns {@code asBitmap(array, Frame.DEPTH_UBYTE)}. */
public Bitmap asBitmap(INDArray array) {
return asBitmap(array, Frame.DEPTH_UBYTE);
}
/**
* Converts an INDArray to a Bitmap. Only intended for images with rank 3.
*
* @param array to convert
* @param dataType from JavaCV (DEPTH_FLOAT, DEPTH_UBYTE, etc), or -1 to use same type as the INDArray
* @return data copied to a Frame
*/
public Bitmap asBitmap(INDArray array, int dataType) {
return converter2.convert(asFrame(array, dataType));
}
}
@@ -0,0 +1,114 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.loader;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.datavec.image.data.Image;
import org.datavec.image.transform.ImageTransform;
import org.nd4j.common.resources.Downloader;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.common.util.ArchiveUtils;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.net.URI;
import java.net.URL;
import java.util.Map;
import java.util.Random;
@Slf4j
public abstract class BaseImageLoader implements Serializable {
public enum MultiPageMode {
MINIBATCH, FIRST //, CHANNELS,
}
public static final String[] ALLOWED_FORMATS = {"tif", "jpg", "png", "jpeg", "bmp", "JPEG", "JPG", "TIF", "PNG"};
protected Random rng = new Random(System.currentTimeMillis());
protected long height = -1;
protected long width = -1;
protected long channels = -1;
protected boolean centerCropIfNeeded = false;
protected ImageTransform imageTransform = null;
protected MultiPageMode multiPageMode = null;
public String[] getAllowedFormats() {
return ALLOWED_FORMATS;
}
public abstract INDArray asRowVector(File f) throws IOException;
public abstract INDArray asRowVector(InputStream inputStream) throws IOException;
/** As per {@link #asMatrix(File, boolean)} but NCHW/channels_first format */
public abstract INDArray asMatrix(File f) throws IOException;
/**
* Load an image from a file to an INDArray
* @param f File to load the image from
* @param nchw If true: return image in NCHW/channels_first [1, channels, height width] format; if false, return
* in NHWC/channels_last [1, height, width, channels] format
* @return Image file as as INDArray
*/
public abstract INDArray asMatrix(File f, boolean nchw) throws IOException;
public abstract INDArray asMatrix(InputStream inputStream) throws IOException;
/**
* Load an image file from an input stream to an INDArray
* @param inputStream Input stream to load the image from
* @param nchw If true: return image in NCHW/channels_first [1, channels, height width] format; if false, return
* in NHWC/channels_last [1, height, width, channels] format
* @return Image file stream as as INDArray
*/
public abstract INDArray asMatrix(InputStream inputStream, boolean nchw) throws IOException;
/** As per {@link #asMatrix(File)} but as an {@link Image}*/
public abstract Image asImageMatrix(File f) throws IOException;
/** As per {@link #asMatrix(File, boolean)} but as an {@link Image}*/
public abstract Image asImageMatrix(File f, boolean nchw) throws IOException;
/** As per {@link #asMatrix(InputStream)} but as an {@link Image}*/
public abstract Image asImageMatrix(InputStream inputStream) throws IOException;
/** As per {@link #asMatrix(InputStream, boolean)} but as an {@link Image}*/
public abstract Image asImageMatrix(InputStream inputStream, boolean nchw) throws IOException;
public static void downloadAndUntar(Map urlMap, File fullDir) {
try {
File file = new File(fullDir, urlMap.get("filesFilename").toString());
if (!file.isFile()) {
Downloader.downloadAndExtract(urlMap.get("filesFilename").toString(),
URI.create(urlMap.get("filesURL").toString()).toURL(),
file,fullDir,"",
3);
}
} catch (IOException e) {
throw new IllegalStateException("Unable to fetch images", e);
}
}
}
@@ -0,0 +1,459 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.loader;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.eclipse.deeplearning4j.resources.DataSetResource;
import org.eclipse.deeplearning4j.resources.ResourceDataSets;
import org.nd4j.linalg.api.ops.impl.reduce.same.Sum;
import org.nd4j.common.primitives.Pair;
import org.datavec.image.data.ImageWritable;
import org.datavec.image.transform.ColorConversionTransform;
import org.datavec.image.transform.EqualizeHistTransform;
import org.datavec.image.transform.ImageTransform;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.util.FeatureUtil;
import java.io.*;
import java.nio.ByteBuffer;
import java.util.*;
import org.bytedeco.opencv.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_imgproc.*;
@Slf4j
public class CifarLoader extends NativeImageLoader implements Serializable {
public static final int NUM_TRAIN_IMAGES = 50000;
public static final int NUM_TEST_IMAGES = 10000;
public static final int NUM_LABELS = 10; // Note 6000 imgs per class
public static final int HEIGHT = 32;
public static final int WIDTH = 32;
public static final int CHANNELS = 3;
public static final boolean DEFAULT_USE_SPECIAL_PREPROC = false;
public static final boolean DEFAULT_SHUFFLE = true;
private static final int BYTEFILELEN = 3073;
private static final String[] TRAINFILENAMES =
{"data_batch_1.bin", "data_batch_2.bin", "data_batch_3.bin", "data_batch_4.bin", "data_batch5.bin"};
private static final String TESTFILENAME = "test_batch.bin";
private static final String labelFileName = "batches.meta.txt";
private static final int numToConvertDS = 10000; // Each file is 10000 images, limiting for file preprocess load
protected final File fullDir;
protected final File meanVarPath;
protected final String trainFilesSerialized;
protected final String testFilesSerialized;
protected InputStream inputStream;
protected InputStream trainInputStream;
protected InputStream testInputStream;
protected List<String> labels = new ArrayList<>();
public static Map<String, String> cifarDataMap = new HashMap<>();
protected boolean train;
protected boolean useSpecialPreProcessCifar;
protected long seed;
protected boolean shuffle = true;
protected int numExamples = 0;
protected double uMean = 0;
protected double uStd = 0;
protected double vMean = 0;
protected double vStd = 0;
protected boolean meanStdStored = false;
protected int loadDSIndex = 0;
protected DataSet loadDS = new DataSet();
protected int fileNum = 0;
private static DataSetResource cifar = ResourceDataSets.cifar10();
private static File getDefaultDirectory() {
return cifar.localCacheDirectory();
}
public CifarLoader() {
this(true);
}
public CifarLoader(boolean train) {
this(train, null);
}
public CifarLoader(boolean train, File fullPath) {
this(HEIGHT, WIDTH, CHANNELS, null, train, DEFAULT_USE_SPECIAL_PREPROC, fullPath, System.currentTimeMillis(),
DEFAULT_SHUFFLE);
}
public CifarLoader(int height, int width, int channels, boolean train, boolean useSpecialPreProcessCifar) {
this(height, width, channels, null, train, useSpecialPreProcessCifar);
}
public CifarLoader(int height, int width, int channels, ImageTransform imgTransform, boolean train,
boolean useSpecialPreProcessCifar) {
this(height, width, channels, imgTransform, train, useSpecialPreProcessCifar, DEFAULT_SHUFFLE);
}
public CifarLoader(int height, int width, int channels, ImageTransform imgTransform, boolean train,
boolean useSpecialPreProcessCifar, boolean shuffle) {
this(height, width, channels, imgTransform, train, useSpecialPreProcessCifar, null, System.currentTimeMillis(),
shuffle);
}
public CifarLoader(int height, int width, int channels, ImageTransform imgTransform, boolean train,
boolean useSpecialPreProcessCifar, File fullDir, long seed, boolean shuffle) {
super(height, width, channels, imgTransform);
this.height = height;
this.width = width;
this.channels = channels;
this.train = train;
this.useSpecialPreProcessCifar = useSpecialPreProcessCifar;
this.seed = seed;
this.shuffle = shuffle;
if (fullDir == null) {
this.fullDir = getDefaultDirectory();
} else {
this.fullDir = fullDir;
}
meanVarPath = new File(this.fullDir, "meanVarPath.txt");
trainFilesSerialized = FilenameUtils.concat(this.fullDir.toString(), "cifar_train_serialized");
testFilesSerialized = FilenameUtils.concat(this.fullDir.toString(), "cifar_test_serialized.ser");
load();
}
@Override
public INDArray asRowVector(File f) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public INDArray asRowVector(InputStream inputStream) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public INDArray asMatrix(File f) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public INDArray asMatrix(InputStream inputStream) throws IOException {
throw new UnsupportedOperationException();
}
private void defineLabels() {
try {
File path = new File(fullDir, labelFileName);
BufferedReader br = new BufferedReader(new FileReader(path));
String line;
while ((line = br.readLine()) != null) {
labels.add(line);
}
} catch (IOException e) {
log.error("",e);
}
}
protected void load() {
if (!cifarRawFilesExist() && !fullDir.exists()) {
fullDir.mkdir();
log.info("Downloading CIFAR data set");
cifar.download(true,3,10000,100000);
}
try {
Collection<File> subFiles = FileUtils.listFiles(fullDir, new String[] {"bin"}, true);
Iterator<File> trainIter = subFiles.iterator();
trainInputStream = new SequenceInputStream(new FileInputStream(trainIter.next()),
new FileInputStream(trainIter.next()));
while (trainIter.hasNext()) {
File nextFile = trainIter.next();
if (!TESTFILENAME.equals(nextFile.getName()))
trainInputStream = new SequenceInputStream(trainInputStream, new FileInputStream(nextFile));
}
testInputStream = new FileInputStream(new File(fullDir, TESTFILENAME));
} catch (Exception e) {
throw new RuntimeException(e);
}
if (labels.isEmpty())
defineLabels();
if (useSpecialPreProcessCifar && train && !cifarProcessedFilesExists()) {
for (int i = fileNum + 1; i <= (TRAINFILENAMES.length); i++) {
inputStream = trainInputStream;
DataSet result = convertDataSet(numToConvertDS);
result.save(new File(trainFilesSerialized + i + ".ser"));
}
// for (int i = 1; i <= (TRAINFILENAMES.length); i++){
// normalizeCifar(new File(trainFilesSerialized + i + ".ser"));
// }
inputStream = testInputStream;
DataSet result = convertDataSet(numToConvertDS);
result.save(new File(testFilesSerialized));
// normalizeCifar(new File(testFilesSerialized));
}
setInputStream();
}
private boolean cifarRawFilesExist() {
File f = new File(fullDir, TESTFILENAME);
if (!f.exists())
return false;
for (String name : TRAINFILENAMES) {
f = new File(fullDir, name);
if (!f.exists())
return false;
}
return true;
}
private boolean cifarProcessedFilesExists() {
File f;
if (train) {
f = new File(trainFilesSerialized + 1 + ".ser");
if (!f.exists())
return false;
} else {
f = new File(testFilesSerialized);
if (!f.exists())
return false;
}
return true;
}
/**
* Preprocess and store cifar based on successful Torch approach by Sergey Zagoruyko
* Reference: <a href="https://github.com/szagoruyko/cifar.torch">https://github.com/szagoruyko/cifar.torch</a>
*/
public Mat convertCifar(Mat orgImage) {
numExamples++;
Mat resImage = new Mat();
OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat();
// ImageTransform yuvTransform = new ColorConversionTransform(new Random(seed), COLOR_BGR2Luv);
// ImageTransform histEqualization = new EqualizeHistTransform(new Random(seed), COLOR_BGR2Luv);
ImageTransform yuvTransform = new ColorConversionTransform(new Random(seed), COLOR_BGR2YCrCb);
ImageTransform histEqualization = new EqualizeHistTransform(new Random(seed), COLOR_BGR2YCrCb);
if (converter != null) {
ImageWritable writable = new ImageWritable(converter.convert(orgImage));
// TODO determine if need to normalize y before transform - opencv docs rec but currently doing after
writable = yuvTransform.transform(writable); // Converts to chrome color to help emphasize image objects
writable = histEqualization.transform(writable); // Normalizes values to further clarify object of interest
resImage = converter.convert(writable.getFrame());
}
return resImage;
}
/**
* Normalize and store cifar based on successful Torch approach by Sergey Zagoruyko
* Reference: <a href="https://github.com/szagoruyko/cifar.torch">https://github.com/szagoruyko/cifar.torch</a>
*/
public void normalizeCifar(File fileName) {
DataSet result = new DataSet();
result.load(fileName);
if (!meanStdStored && train) {
uMean = Math.abs(uMean / numExamples);
uStd = Math.sqrt(uStd);
vMean = Math.abs(vMean / numExamples);
vStd = Math.sqrt(vStd);
// TODO find cleaner way to store and load (e.g. json or yaml)
try {
FileUtils.write(meanVarPath, uMean + "," + uStd + "," + vMean + "," + vStd);
} catch (IOException e) {
log.error("",e);
}
meanStdStored = true;
} else if (uMean == 0 && meanStdStored) {
try {
String[] values = FileUtils.readFileToString(meanVarPath).split(",");
uMean = Double.parseDouble(values[0]);
uStd = Double.parseDouble(values[1]);
vMean = Double.parseDouble(values[2]);
vStd = Double.parseDouble(values[3]);
} catch (IOException e) {
log.error("",e);
}
}
for (int i = 0; i < result.numExamples(); i++) {
INDArray newFeatures = result.get(i).getFeatures();
newFeatures.tensorAlongDimension(0, new long[] {0, 2, 3}).divi(255);
newFeatures.tensorAlongDimension(1, new long[] {0, 2, 3}).subi(uMean).divi(uStd);
newFeatures.tensorAlongDimension(2, new long[] {0, 2, 3}).subi(vMean).divi(vStd);
result.get(i).setFeatures(newFeatures);
}
result.save(fileName);
}
public Pair<INDArray, Mat> convertMat(byte[] byteFeature) {
INDArray label = FeatureUtil.toOutcomeVector(byteFeature[0], NUM_LABELS);// first value in the 3073 byte array
Mat image = new Mat(HEIGHT, WIDTH, CV_8UC(CHANNELS)); // feature are 3072
ByteBuffer imageData = image.createBuffer();
for (int i = 0; i < HEIGHT * WIDTH; i++) {
imageData.put(3 * i, byteFeature[i + 1 + 2 * HEIGHT * WIDTH]); // blue
imageData.put(3 * i + 1, byteFeature[i + 1 + HEIGHT * WIDTH]); // green
imageData.put(3 * i + 2, byteFeature[i + 1]); // red
}
// if (useSpecialPreProcessCifar) {
// image = convertCifar(image);
// }
return new Pair<>(label, image);
}
public DataSet convertDataSet(int num) {
int batchNumCount = 0;
List<DataSet> dataSets = new ArrayList<>();
Pair<INDArray, Mat> matConversion;
byte[] byteFeature = new byte[BYTEFILELEN];
try {
// while (inputStream.read(byteFeature) != -1 && batchNumCount != num) {
while (batchNumCount != num && inputStream.read(byteFeature) != -1 ) {
matConversion = convertMat(byteFeature);
try {
dataSets.add(new DataSet(asMatrix(matConversion.getSecond()), matConversion.getFirst()));
batchNumCount++;
} catch (Exception e) {
log.error("",e);
break;
}
}
} catch (IOException e) {
log.error("",e);
}
if(dataSets.size() == 0){
return new DataSet();
}
DataSet result = DataSet.merge(dataSets);
double uTempMean, vTempMean;
for (DataSet data : result) {
try {
if (useSpecialPreProcessCifar) {
INDArray uChannel = data.getFeatures().tensorAlongDimension(1, new long[] {0, 2, 3});
INDArray vChannel = data.getFeatures().tensorAlongDimension(2, new long[] {0, 2, 3});
uTempMean = uChannel.meanNumber().doubleValue();
// TODO INDArray.var result is incorrect based on dimensions passed in thus using manual
uStd += varManual(uChannel, uTempMean);
uMean += uTempMean;
vTempMean = vChannel.meanNumber().doubleValue();
vStd += varManual(vChannel, vTempMean);
vMean += vTempMean;
data.setFeatures(data.getFeatures().div(255));
} else {
// normalize if just input stream and not special preprocess
data.setFeatures(data.getFeatures().div(255));
}
} catch (IllegalArgumentException e) {
throw new IllegalStateException("The number of channels must be 3 to special preProcess Cifar with.");
}
}
if (shuffle && num > 1)
result.shuffle(seed);
return result;
}
public double varManual(INDArray x, double mean) {
INDArray xSubMean = x.sub(mean);
INDArray squared = xSubMean.muli(xSubMean);
double accum = Nd4j.getExecutioner().execAndReturn(new Sum(squared)).getFinalResult().doubleValue();
return accum / x.ravel().length();
}
public DataSet next(int batchSize) {
return next(batchSize, 0);
}
public DataSet next(int batchSize, int exampleNum) {
List<DataSet> temp = new ArrayList<>();
DataSet result;
if (cifarProcessedFilesExists() && useSpecialPreProcessCifar) {
if (exampleNum == 0 || ((exampleNum / fileNum) == numToConvertDS && train)) {
fileNum++;
if (train)
loadDS.load(new File(trainFilesSerialized + fileNum + ".ser"));
loadDS.load(new File(testFilesSerialized));
// Shuffle all examples in file before batching happens also for each reset
if (shuffle && batchSize > 1)
loadDS.shuffle(seed);
loadDSIndex = 0;
// inputBatched = loadDS.batchBy(batchSize);
}
// TODO loading full train dataset when using cuda causes memory error - find way to load into list off gpu
// result = inputBatched.get(batchNum);
for (int i = 0; i < batchSize; i++) {
if (loadDS.get(loadDSIndex) != null)
temp.add(loadDS.get(loadDSIndex));
else
break;
loadDSIndex++;
}
if (temp.size() > 1)
result = DataSet.merge(temp);
else
result = temp.get(0);
} else {
result = convertDataSet(batchSize);
}
return result;
}
public InputStream getInputStream() {
return inputStream;
}
public void setInputStream() {
if (train)
inputStream = trainInputStream;
else
inputStream = testInputStream;
}
public List<String> getLabels() {
return labels;
}
public void reset() {
numExamples = 0;
fileNum = 0;
load();
}
}
@@ -0,0 +1,572 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.loader;
import com.github.jaiimageio.impl.plugins.tiff.TIFFImageReaderSpi;
import com.github.jaiimageio.impl.plugins.tiff.TIFFImageWriterSpi;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.common.util.ArrayUtil;
import org.nd4j.linalg.util.NDArrayUtil;
import javax.imageio.ImageIO;
import javax.imageio.spi.IIORegistry;
import java.awt.*;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferByte;
import java.awt.image.Raster;
import java.awt.image.WritableRaster;
import java.io.*;
import java.util.Arrays;
public class ImageLoader extends BaseImageLoader {
static {
ImageIO.scanForPlugins();
IIORegistry registry = IIORegistry.getDefaultInstance();
registry.registerServiceProvider(new TIFFImageWriterSpi());
registry.registerServiceProvider(new TIFFImageReaderSpi());
registry.registerServiceProvider(new com.twelvemonkeys.imageio.plugins.jpeg.JPEGImageReaderSpi());
registry.registerServiceProvider(new com.twelvemonkeys.imageio.plugins.jpeg.JPEGImageWriterSpi());
registry.registerServiceProvider(new com.twelvemonkeys.imageio.plugins.psd.PSDImageReaderSpi());
registry.registerServiceProvider(Arrays.asList(new com.twelvemonkeys.imageio.plugins.bmp.BMPImageReaderSpi(),
new com.twelvemonkeys.imageio.plugins.bmp.CURImageReaderSpi(),
new com.twelvemonkeys.imageio.plugins.bmp.ICOImageReaderSpi()));
}
public ImageLoader() {
super();
}
/**
* Instantiate an image with the given
* height and width
*
* @param height the height to load*
* @param width the width to load
*/
public ImageLoader(long height, long width) {
super();
this.height = height;
this.width = width;
}
/**
* Instantiate an image with the given
* height and width
*
* @param height the height to load
* @param width the width to load
* @param channels the number of channels for the image*
*/
public ImageLoader(long height, long width, long channels) {
super();
this.height = height;
this.width = width;
this.channels = channels;
}
/**
* Instantiate an image with the given
* height and width
*
* @param height the height to load
* @param width the width to load
* @param channels the number of channels for the image*
* @param centerCropIfNeeded to crop before rescaling and converting
*/
public ImageLoader(long height, long width, long channels, boolean centerCropIfNeeded) {
this(height, width, channels);
this.centerCropIfNeeded = centerCropIfNeeded;
}
/**
* Convert a file to a row vector
*
* @param f the image to convert
* @return the flattened image
* @throws IOException
*/
public INDArray asRowVector(File f) throws IOException {
return asRowVector(ImageIO.read(f));
// if(channels == 3) {
// return toRaveledTensor(f);
// }
// return NDArrayUtil.toNDArray(flattenedImageFromFile(f));
}
public INDArray asRowVector(InputStream inputStream) throws IOException {
return asRowVector(ImageIO.read(inputStream));
// return asMatrix(inputStream).ravel();
}
/**
* Convert an image in to a row vector
*
* @param image the image to convert
* @return the row vector based on a rastered
* representation of the image
*/
public INDArray asRowVector(BufferedImage image) {
if (centerCropIfNeeded) {
image = centerCropIfNeeded(image);
}
image = scalingIfNeed(image, true);
if (channels == 3) {
return toINDArrayBGR(image).ravel();
}
int[][] ret = toIntArrayArray(image);
return NDArrayUtil.toNDArray(ArrayUtil.flatten(ret));
}
/**
* Changes the input stream in to an
* bgr based raveled(flattened) vector
*
* @param file the input stream to convert
* @return the raveled bgr values for this input stream
*/
public INDArray toRaveledTensor(File file) {
try {
BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file));
INDArray ret = toRaveledTensor(bis);
bis.close();
return ret.ravel();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Changes the input stream in to an
* bgr based raveled(flattened) vector
*
* @param is the input stream to convert
* @return the raveled bgr values for this input stream
*/
public INDArray toRaveledTensor(InputStream is) {
return toBgr(is).ravel();
}
/**
* Convert an image in to a raveled tensor of
* the bgr values of the image
*
* @param image the image to parse
* @return the raveled tensor of bgr values
*/
public INDArray toRaveledTensor(BufferedImage image) {
try {
image = scalingIfNeed(image, false);
return toINDArrayBGR(image).ravel();
} catch (Exception e) {
throw new RuntimeException("Unable to load image", e);
}
}
/**
* Convert an input stream to an bgr spectrum image
*
* @param file the file to convert
* @return the input stream to convert
*/
public INDArray toBgr(File file) {
try {
BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file));
INDArray ret = toBgr(bis);
bis.close();
return ret;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Convert an input stream to an bgr spectrum image
*
* @param inputStream the input stream to convert
* @return the input stream to convert
*/
public INDArray toBgr(InputStream inputStream) {
try {
BufferedImage image = ImageIO.read(inputStream);
return toBgr(image);
} catch (IOException e) {
throw new RuntimeException("Unable to load image", e);
}
}
private org.datavec.image.data.Image toBgrImage(InputStream inputStream) {
try {
BufferedImage image = ImageIO.read(inputStream);
INDArray img = toBgr(image);
return new org.datavec.image.data.Image(img, image.getData().getNumBands(), image.getHeight(), image.getWidth());
} catch (IOException e) {
throw new RuntimeException("Unable to load image", e);
}
}
/**
* Convert an BufferedImage to an bgr spectrum image
*
* @param image the BufferedImage to convert
* @return the input stream to convert
*/
public INDArray toBgr(BufferedImage image) {
if (image == null)
throw new IllegalStateException("Unable to load image");
image = scalingIfNeed(image, false);
return toINDArrayBGR(image);
}
/**
* Convert an image file
* in to a matrix
*
* @param f the file to convert
* @return a 2d matrix of a rastered version of the image
* @throws IOException
*/
public INDArray asMatrix(File f) throws IOException {
return asMatrix(f, true);
}
@Override
public INDArray asMatrix(File f, boolean nchw) throws IOException {
try(InputStream is = new BufferedInputStream(new FileInputStream(f))){
return asMatrix(is, nchw);
}
}
/**
* Convert an input stream to a matrix
*
* @param inputStream the input stream to convert
* @return the input stream to convert
*/
public INDArray asMatrix(InputStream inputStream) throws IOException {
return asMatrix(inputStream, true);
}
@Override
public INDArray asMatrix(InputStream inputStream, boolean nchw) throws IOException {
INDArray ret;
if (channels == 3) {
ret = toBgr(inputStream);
} else {
try {
BufferedImage image = ImageIO.read(inputStream);
ret = asMatrix(image);
} catch (IOException e) {
throw new IOException("Unable to load image", e);
}
}
if(ret.rank() == 3){
ret = ret.reshape(1, ret.size(0), ret.size(1), ret.size(2));
}
if(!nchw)
ret = ret.permute(0,2,3,1); //NCHW to NHWC
return ret;
}
@Override
public org.datavec.image.data.Image asImageMatrix(File f) throws IOException {
return asImageMatrix(f, true);
}
@Override
public org.datavec.image.data.Image asImageMatrix(File f, boolean nchw) throws IOException {
try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f))) {
return asImageMatrix(bis, nchw);
}
}
@Override
public org.datavec.image.data.Image asImageMatrix(InputStream inputStream) throws IOException {
return asImageMatrix(inputStream, true);
}
@Override
public org.datavec.image.data.Image asImageMatrix(InputStream inputStream, boolean nchw) throws IOException {
org.datavec.image.data.Image ret;
if (channels == 3) {
ret = toBgrImage(inputStream);
} else {
try {
BufferedImage image = ImageIO.read(inputStream);
INDArray asMatrix = asMatrix(image);
ret = new org.datavec.image.data.Image(asMatrix, image.getData().getNumBands(), image.getHeight(), image.getWidth());
} catch (IOException e) {
throw new IOException("Unable to load image", e);
}
}
if(ret.getImage().rank() == 3){
INDArray a = ret.getImage();
ret.setImage(a.reshape(1, a.size(0), a.size(1), a.size(2)));
}
if(!nchw)
ret.setImage(ret.getImage().permute(0,2,3,1)); //NCHW to NHWC
return ret;
}
/**
* Convert an BufferedImage to a matrix
*
* @param image the BufferedImage to convert
* @return the input stream to convert
*/
public INDArray asMatrix(BufferedImage image) {
if (channels == 3) {
return toBgr(image);
} else {
image = scalingIfNeed(image, true);
int w = image.getWidth();
int h = image.getHeight();
INDArray ret = Nd4j.create(h, w);
for (int i = 0; i < h; i++) {
for (int j = 0; j < w; j++) {
ret.putScalar(new int[]{i, j}, image.getRGB(j, i));
}
}
return ret;
}
}
/**
* Slices up an image in to a mini batch.
*
* @param f the file to load from
* @param numMiniBatches the number of images in a mini batch
* @param numRowsPerSlice the number of rows for each image
* @return a tensor representing one image as a mini batch
*/
public INDArray asImageMiniBatches(File f, int numMiniBatches, int numRowsPerSlice) {
try {
INDArray d = asMatrix(f);
return Nd4j.create(numMiniBatches, numRowsPerSlice, d.columns());
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public int[] flattenedImageFromFile(File f) throws IOException {
return ArrayUtil.flatten(fromFile(f));
}
/**
* Load a rastered image from file
*
* @param file the file to load
* @return the rastered image
* @throws IOException
*/
public int[][] fromFile(File file) throws IOException {
BufferedImage image = ImageIO.read(file);
image = scalingIfNeed(image, true);
return toIntArrayArray(image);
}
/**
* Load a rastered image from file
*
* @param file the file to load
* @return the rastered image
* @throws IOException
*/
public int[][][] fromFileMultipleChannels(File file) throws IOException {
BufferedImage image = ImageIO.read(file);
image = scalingIfNeed(image, channels > 3);
int w = image.getWidth(), h = image.getHeight();
int bands = image.getSampleModel().getNumBands();
int[][][] ret = new int[(int) Math.min(channels, Integer.MAX_VALUE)]
[(int) Math.min(h, Integer.MAX_VALUE)]
[(int) Math.min(w, Integer.MAX_VALUE)];
byte[] pixels = ((DataBufferByte) image.getRaster().getDataBuffer()).getData();
for (int i = 0; i < h; i++) {
for (int j = 0; j < w; j++) {
for (int k = 0; k < channels; k++) {
if (k >= bands)
break;
ret[k][i][j] = pixels[(int) Math.min(channels * w * i + channels * j + k, Integer.MAX_VALUE)];
}
}
}
return ret;
}
/**
* Convert a matrix in to a buffereed image
*
* @param matrix the
* @return {@link java.awt.image.BufferedImage}
*/
public static BufferedImage toImage(INDArray matrix) {
BufferedImage img = new BufferedImage(matrix.rows(), matrix.columns(), BufferedImage.TYPE_INT_ARGB);
WritableRaster r = img.getRaster();
int[] equiv = new int[(int) matrix.length()];
for (int i = 0; i < equiv.length; i++) {
equiv[i] = (int) matrix.getDouble(i);
}
r.setDataElements(0, 0, matrix.rows(), matrix.columns(), equiv);
return img;
}
private static int[] rasterData(INDArray matrix) {
int[] ret = new int[(int) matrix.length()];
for (int i = 0; i < ret.length; i++)
ret[i] = (int) Math.round((double) matrix.getScalar(i).element());
return ret;
}
/**
* Convert the given image to an rgb image
*
* @param arr the array to use
* @param image the image to set
*/
public void toBufferedImageRGB(INDArray arr, BufferedImage image) {
if (arr.rank() < 3)
throw new IllegalArgumentException("Arr must be 3d");
image = scalingIfNeed(image, arr.size(-2), arr.size(-1), image.getType(), true);
for (int i = 0; i < image.getHeight(); i++) {
for (int j = 0; j < image.getWidth(); j++) {
int r = arr.slice(2).getInt(i, j);
int g = arr.slice(1).getInt(i, j);
int b = arr.slice(0).getInt(i, j);
int a = 1;
int col = (a << 24) | (r << 16) | (g << 8) | b;
image.setRGB(j, i, col);
}
}
}
/**
* Converts a given Image into a BufferedImage
*
* @param img The Image to be converted
* @param type The color model of BufferedImage
* @return The converted BufferedImage
*/
public static BufferedImage toBufferedImage(Image img, int type) {
if (img instanceof BufferedImage && ((BufferedImage) img).getType() == type) {
return (BufferedImage) img;
}
// Create a buffered image with transparency
BufferedImage bimage = new BufferedImage(img.getWidth(null), img.getHeight(null), type);
// Draw the image on to the buffered image
Graphics2D bGr = bimage.createGraphics();
bGr.drawImage(img, 0, 0, null);
bGr.dispose();
// Return the buffered image
return bimage;
}
protected int[][] toIntArrayArray(BufferedImage image) {
int w = image.getWidth(), h = image.getHeight();
int[][] ret = new int[h][w];
if (image.getRaster().getNumDataElements() == 1) {
Raster raster = image.getRaster();
for (int i = 0; i < h; i++) {
for (int j = 0; j < w; j++) {
ret[i][j] = raster.getSample(j, i, 0);
}
}
} else {
for (int i = 0; i < h; i++) {
for (int j = 0; j < w; j++) {
ret[i][j] = image.getRGB(j, i);
}
}
}
return ret;
}
protected INDArray toINDArrayBGR(BufferedImage image) {
int height = image.getHeight();
int width = image.getWidth();
int bands = image.getSampleModel().getNumBands();
byte[] pixels = ((DataBufferByte) image.getRaster().getDataBuffer()).getData();
int[] shape = new int[]{height, width, bands};
INDArray ret2 = Nd4j.create(1, pixels.length);
for (int i = 0; i < ret2.length(); i++) {
ret2.putScalar(i, ((int) pixels[i]) & 0xFF);
}
return ret2.reshape(shape).permute(2, 0, 1);
}
// TODO build flexibility on where to crop the image
public BufferedImage centerCropIfNeeded(BufferedImage img) {
int x = 0;
int y = 0;
int height = img.getHeight();
int width = img.getWidth();
int diff = Math.abs(width - height) / 2;
if (width > height) {
x = diff;
width = width - diff;
} else if (height > width) {
y = diff;
height = height - diff;
}
return img.getSubimage(x, y, width, height);
}
protected BufferedImage scalingIfNeed(BufferedImage image, boolean needAlpha) {
return scalingIfNeed(image, height, width, channels, needAlpha);
}
protected BufferedImage scalingIfNeed(BufferedImage image, long dstHeight, long dstWidth, long dstImageType, boolean needAlpha) {
Image scaled;
// Scale width and height first if necessary
if (dstHeight > 0 && dstWidth > 0 && (image.getHeight() != dstHeight || image.getWidth() != dstWidth)) {
scaled = image.getScaledInstance((int) dstWidth, (int) dstHeight, Image.SCALE_SMOOTH);
} else {
scaled = image;
}
// Transfer imageType if necessary and transfer to BufferedImage.
if (scaled instanceof BufferedImage && ((BufferedImage) scaled).getType() == dstImageType) {
return (BufferedImage) scaled;
}
if (needAlpha && image.getColorModel().hasAlpha() && dstImageType == BufferedImage.TYPE_4BYTE_ABGR) {
return toBufferedImage(scaled, BufferedImage.TYPE_4BYTE_ABGR);
} else {
if (dstImageType == BufferedImage.TYPE_BYTE_GRAY)
return toBufferedImage(scaled, BufferedImage.TYPE_BYTE_GRAY);
else
return toBufferedImage(scaled, BufferedImage.TYPE_3BYTE_BGR);
}
}
}
@@ -0,0 +1,119 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.loader;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.Java2DFrameConverter;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.datavec.image.transform.ImageTransform;
import org.nd4j.linalg.api.ndarray.INDArray;
import java.awt.image.BufferedImage;
import java.io.IOException;
public class Java2DNativeImageLoader extends NativeImageLoader {
Java2DFrameConverter converter2 = new Java2DFrameConverter();
public Java2DNativeImageLoader() {}
public Java2DNativeImageLoader(int height, int width) {
super(height, width);
}
public Java2DNativeImageLoader(int height, int width, int channels) {
super(height, width, channels);
}
public Java2DNativeImageLoader(int height, int width, int channels, boolean centerCropIfNeeded) {
super(height, width, channels, centerCropIfNeeded);
}
public Java2DNativeImageLoader(int height, int width, int channels, ImageTransform imageTransform) {
super(height, width, channels, imageTransform);
}
protected Java2DNativeImageLoader(NativeImageLoader other) {
super(other);
}
/**
* Returns {@code asMatrix(image, false).ravel()}.
*/
public INDArray asRowVector(BufferedImage image) throws IOException {
return asMatrix(image, false).ravel();
}
/**
* Returns {@code asMatrix(image, false)}.
*/
public INDArray asMatrix(BufferedImage image) throws IOException {
return asMatrix(image, false);
}
/**
* Returns {@code asMatrix(image, flipChannels).ravel()}.
*/
public INDArray asRowVector(BufferedImage image, boolean flipChannels) throws IOException {
return asMatrix(image, flipChannels).ravel();
}
/**
* Loads a {@link INDArray} from a {@link BufferedImage}.
*
* @param image as a BufferedImage
* @param flipChannels to have a format like TYPE_INT_RGB (ARGB) output as BGRA, etc
* @return the loaded matrix
* @throws IOException
*/
public INDArray asMatrix(BufferedImage image, boolean flipChannels) throws IOException {
if (converter == null) {
converter = new OpenCVFrameConverter.ToMat();
}
return asMatrix(converter.convert(converter2.getFrame(image, 1.0, flipChannels)));
}
@Override
public INDArray asRowVector(Object image) throws IOException {
return image instanceof BufferedImage ? asRowVector((BufferedImage) image) : null;
}
@Override
public INDArray asMatrix(Object image) throws IOException {
return image instanceof BufferedImage ? asMatrix((BufferedImage) image) : null;
}
/** Returns {@code asBufferedImage(array, Frame.DEPTH_UBYTE)}. */
public BufferedImage asBufferedImage(INDArray array) {
return asBufferedImage(array, Frame.DEPTH_UBYTE);
}
/**
* Converts an INDArray to a BufferedImage. Only intended for images with rank 3.
*
* @param array to convert
* @param dataType from JavaCV (DEPTH_FLOAT, DEPTH_UBYTE, etc), or -1 to use same type as the INDArray
* @return data copied to a Frame
*/
public BufferedImage asBufferedImage(INDArray array, int dataType) {
return converter2.convert(asFrame(array, dataType));
}
}
@@ -0,0 +1,275 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.loader;
import lombok.extern.slf4j.Slf4j;
import org.datavec.api.io.filters.BalancedPathFilter;
import org.datavec.api.io.labels.PathLabelGenerator;
import org.datavec.api.io.labels.PatternPathLabelGenerator;
import org.datavec.api.records.reader.RecordReader;
import org.datavec.api.split.FileSplit;
import org.datavec.api.split.InputSplit;
import org.datavec.image.data.Image;
import org.datavec.image.recordreader.ImageRecordReader;
import org.datavec.image.transform.ImageTransform;
import org.eclipse.deeplearning4j.resources.DataSetResource;
import org.eclipse.deeplearning4j.resources.ResourceDataSets;
import org.nd4j.linalg.api.ndarray.INDArray;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
@Slf4j
public class LFWLoader extends BaseImageLoader implements Serializable {
public final static int NUM_IMAGES = 13233;
public final static int NUM_LABELS = 5749;
public final static int SUB_NUM_IMAGES = 1054;
public final static int SUB_NUM_LABELS = 432;
public final static int HEIGHT = 250;
public final static int WIDTH = 250;
public final static int CHANNELS = 3;
public final static String DATA_URL = "http://vis-www.cs.umass.edu/lfw/lfw.tgz";
public final static String LABEL_URL = "http://vis-www.cs.umass.edu/lfw/lfw-names.txt";
public final static String SUBSET_URL = "http://vis-www.cs.umass.edu/lfw/lfw-a.tgz";
protected final static String REGEX_PATTERN = ".[0-9]+";
public final static PathLabelGenerator LABEL_PATTERN = new PatternPathLabelGenerator(REGEX_PATTERN);
public String dataFile = "lfw";
public String labelFile = "lfw-names.txt";
public String subsetFile = "lfw-a";
private static DataSetResource lfwFull = ResourceDataSets.lfwFullData();
private static DataSetResource lfwSub = ResourceDataSets.lfwSubData();
private static DataSetResource lfwLabels = ResourceDataSets.lfwFullData();
protected boolean useSubset = false;
protected InputSplit[] inputSplit;
public LFWLoader() {
this(false);
}
public LFWLoader(boolean useSubset) {
this(new long[] {HEIGHT, WIDTH, CHANNELS,}, null, useSubset);
}
public LFWLoader(int[] imgDim, boolean useSubset) {
this(imgDim, null, useSubset);
}
public LFWLoader(long[] imgDim, boolean useSubset) {
this(imgDim, null, useSubset);
}
public LFWLoader(int[] imgDim, ImageTransform imgTransform, boolean useSubset) {
this.height = imgDim[0];
this.width = imgDim[1];
this.channels = imgDim[2];
this.imageTransform = imgTransform;
this.useSubset = useSubset;
}
public LFWLoader(long[] imgDim, ImageTransform imgTransform, boolean useSubset) {
this.height = imgDim[0];
this.width = imgDim[1];
this.channels = imgDim[2];
this.imageTransform = imgTransform;
this.useSubset = useSubset;
}
public void load() {
load(NUM_IMAGES, NUM_IMAGES, NUM_LABELS, LABEL_PATTERN, 1, rng);
}
public void load(long batchSize, long numExamples, long numLabels, PathLabelGenerator labelGenerator,
double splitTrainTest, Random rng) {
if (!imageFilesExist()) {
if (useSubset) {
lfwSub.download(true,3,20000,20000);
lfwLabels.download(true,3,30000,3000);
} else {
lfwFull.download(true,3,20000,20000);
lfwLabels.download(true,3,30000,3000);
}
}
File inputDir = useSubset ? lfwSub.localCacheDirectory() : lfwFull.localCacheDirectory();
FileSplit fileSplit = new FileSplit(inputDir, ALLOWED_FORMATS, rng);
BalancedPathFilter pathFilter = new BalancedPathFilter(rng, ALLOWED_FORMATS, labelGenerator, numExamples,
numLabels, 0, batchSize, null);
inputSplit = fileSplit.sample(pathFilter, numExamples * splitTrainTest, numExamples * (1 - splitTrainTest));
}
public boolean imageFilesExist() {
if (useSubset) {
if (!lfwSub.existsLocally())
return lfwSub.existsLocally();
} else {
return lfwFull.existsLocally();
}
return true;
}
public RecordReader getRecordReader(long numExamples) {
return getRecordReader(numExamples, numExamples, new long[] {height, width, channels},
useSubset ? SUB_NUM_LABELS : NUM_LABELS, LABEL_PATTERN, true, 1,
new Random(System.currentTimeMillis()));
}
public RecordReader getRecordReader(long batchSize, long numExamples, long numLabels, Random rng) {
return getRecordReader(numExamples, batchSize, new long[] {height, width, channels}, numLabels, LABEL_PATTERN,
true, 1, rng);
}
public RecordReader getRecordReader(long batchSize, long numExamples, boolean train, double splitTrainTest) {
return getRecordReader(numExamples, batchSize, new long[] {height, width, channels},
useSubset ? SUB_NUM_LABELS : NUM_LABELS, LABEL_PATTERN, train, splitTrainTest,
new Random(System.currentTimeMillis()));
}
public RecordReader getRecordReader(long batchSize, long numExamples, int[] imgDim, boolean train,
double splitTrainTest, Random rng) {
return getRecordReader(numExamples, batchSize, imgDim, useSubset ? SUB_NUM_LABELS : NUM_LABELS, LABEL_PATTERN,
train, splitTrainTest, rng);
}
public RecordReader getRecordReader(long batchSize, long numExamples, long[] imgDim, boolean train,
double splitTrainTest, Random rng) {
return getRecordReader(numExamples, batchSize, imgDim, useSubset ? SUB_NUM_LABELS : NUM_LABELS, LABEL_PATTERN,
train, splitTrainTest, rng);
}
public RecordReader getRecordReader(long batchSize, long numExamples, PathLabelGenerator labelGenerator,
boolean train, double splitTrainTest, Random rng) {
return getRecordReader(numExamples, batchSize, new long[] {height, width, channels},
useSubset ? SUB_NUM_LABELS : NUM_LABELS, labelGenerator, train, splitTrainTest, rng);
}
public RecordReader getRecordReader(long batchSize, long numExamples, int[] imgDim, PathLabelGenerator labelGenerator,
boolean train, double splitTrainTest, Random rng) {
return getRecordReader(numExamples, batchSize, imgDim, useSubset ? SUB_NUM_LABELS : NUM_LABELS, labelGenerator,
train, splitTrainTest, rng);
}
public RecordReader getRecordReader(long batchSize, long numExamples, long[] imgDim, PathLabelGenerator labelGenerator,
boolean train, double splitTrainTest, Random rng) {
return getRecordReader(numExamples, batchSize, imgDim, useSubset ? SUB_NUM_LABELS : NUM_LABELS, labelGenerator,
train, splitTrainTest, rng);
}
public RecordReader getRecordReader(long batchSize, long numExamples, int[] imgDim, long numLabels,
PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
load(batchSize, numExamples, numLabels, labelGenerator, splitTrainTest, rng);
RecordReader recordReader =
new ImageRecordReader(imgDim[0], imgDim[1], imgDim[2], labelGenerator, imageTransform);
try {
InputSplit data = train ? inputSplit[0] : inputSplit[1];
recordReader.initialize(data);
} catch (IOException | InterruptedException e) {
log.error("",e);
}
return recordReader;
}
public RecordReader getRecordReader(long batchSize, long numExamples, long[] imgDim, long numLabels,
PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
load(batchSize, numExamples, numLabels, labelGenerator, splitTrainTest, rng);
RecordReader recordReader =
new ImageRecordReader(imgDim[0], imgDim[1], imgDim[2], labelGenerator, imageTransform);
try {
InputSplit data = train ? inputSplit[0] : inputSplit[1];
recordReader.initialize(data);
} catch (IOException | InterruptedException e) {
log.error("",e);
}
return recordReader;
}
@Override
public INDArray asRowVector(File f) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public INDArray asRowVector(InputStream inputStream) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public INDArray asMatrix(File f) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public INDArray asMatrix(File f, boolean nchw) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public INDArray asMatrix(InputStream inputStream) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public INDArray asMatrix(InputStream inputStream, boolean nchw) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public Image asImageMatrix(File f) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public Image asImageMatrix(File f, boolean nchw) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public Image asImageMatrix(InputStream inputStream) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public Image asImageMatrix(InputStream inputStream, boolean nchw) throws IOException {
throw new UnsupportedOperationException();
}
}
@@ -0,0 +1,866 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.loader;
import org.apache.commons.io.IOUtils;
import org.bytedeco.javacpp.*;
import org.bytedeco.javacpp.indexer.*;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.datavec.image.data.Image;
import org.datavec.image.data.ImageWritable;
import org.datavec.image.transform.ImageTransform;
import org.nd4j.linalg.api.concurrency.AffinityManager;
import org.nd4j.linalg.api.memory.pointers.PagedPointer;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.exception.ND4JIllegalStateException;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.indexing.INDArrayIndex;
import org.nd4j.linalg.indexing.NDArrayIndex;
import org.nd4j.common.util.ArrayUtil;
import java.io.*;
import java.nio.ByteOrder;
import org.bytedeco.leptonica.*;
import org.bytedeco.opencv.opencv_core.*;
import static org.bytedeco.leptonica.global.leptonica.*;
import static org.bytedeco.opencv.global.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_imgcodecs.*;
import static org.bytedeco.opencv.global.opencv_imgproc.*;
public class NativeImageLoader extends BaseImageLoader {
private static final int MIN_BUFFER_STEP_SIZE = 64 * 1024;
public static final String[] ALLOWED_FORMATS = {"bmp", "gif", "jpg", "jpeg", "jp2", "pbm", "pgm", "ppm", "pnm",
"png", "tif", "tiff", "exr", "webp", "BMP", "GIF", "JPG", "JPEG", "JP2", "PBM", "PGM", "PPM", "PNM",
"PNG", "TIF", "TIFF", "EXR", "WEBP"};
protected OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat();
boolean direct = !Loader.getPlatform().startsWith("android");
/**
* Loads images with no scaling or conversion.
*/
public NativeImageLoader() {}
/**
* Instantiate an image with the given
* height and width
* @param height the height to load
* @param width the width to load
*/
public NativeImageLoader(long height, long width) {
this.height = height;
this.width = width;
}
/**
* Instantiate an image with the given
* height and width
* @param height the height to load
* @param width the width to load
* @param channels the number of channels for the image*
*/
public NativeImageLoader(long height, long width, long channels) {
this.height = height;
this.width = width;
this.channels = channels;
}
/**
* Instantiate an image with the given
* height and width
* @param height the height to load
* @param width the width to load
* @param channels the number of channels for the image*
* @param centerCropIfNeeded to crop before rescaling and converting
*/
public NativeImageLoader(long height, long width, long channels, boolean centerCropIfNeeded) {
this(height, width, channels);
this.centerCropIfNeeded = centerCropIfNeeded;
}
/**
* Instantiate an image with the given
* height and width
* @param height the height to load
* @param width the width to load
* @param channels the number of channels for the image*
* @param imageTransform to use before rescaling and converting
*/
public NativeImageLoader(long height, long width, long channels, ImageTransform imageTransform) {
this(height, width, channels);
this.imageTransform = imageTransform;
}
/**
* Instantiate an image with the given
* height and width
* @param height the height to load
* @param width the width to load
* @param channels the number of channels for the image*
* @param mode how to load multipage image
*/
public NativeImageLoader(long height, long width, long channels, MultiPageMode mode) {
this(height, width, channels);
this.multiPageMode = mode;
}
protected NativeImageLoader(NativeImageLoader other) {
this.height = other.height;
this.width = other.width;
this.channels = other.channels;
this.centerCropIfNeeded = other.centerCropIfNeeded;
this.imageTransform = other.imageTransform;
this.multiPageMode = other.multiPageMode;
}
@Override
public String[] getAllowedFormats() {
return ALLOWED_FORMATS;
}
public INDArray asRowVector(String filename) throws IOException {
return asRowVector(new File(filename));
}
/**
* Convert a file to a row vector
*
* @param f the image to convert
* @return the flattened image
* @throws IOException
*/
@Override
public INDArray asRowVector(File f) throws IOException {
return asMatrix(f).ravel();
}
@Override
public INDArray asRowVector(InputStream is) throws IOException {
return asMatrix(is).ravel();
}
/**
* Returns {@code asMatrix(image).ravel()}.
* @see #asMatrix(Object)
*/
public INDArray asRowVector(Object image) throws IOException {
return asMatrix(image).ravel();
}
public INDArray asRowVector(Frame image) throws IOException {
return asMatrix(image).ravel();
}
public INDArray asRowVector(Mat image) throws IOException {
INDArray arr = asMatrix(image);
return arr.reshape('c', 1, arr.length());
}
public INDArray asRowVector(org.opencv.core.Mat image) throws IOException {
INDArray arr = asMatrix(image);
return arr.reshape('c', 1, arr.length());
}
static Mat convert(PIX pix) {
PIX tempPix = null;
int dtype = -1;
int height = pix.h();
int width = pix.w();
Mat mat2;
if (pix.colormap() != null) {
PIX pix2 = pixRemoveColormap(pix, REMOVE_CMAP_TO_FULL_COLOR);
tempPix = pix = pix2;
dtype = CV_8UC4;
} else if (pix.d() <= 8 || pix.d() == 24) {
PIX pix2 = null;
switch (pix.d()) {
case 1:
pix2 = pixConvert1To8(null, pix, (byte) 0, (byte) 255);
break;
case 2:
pix2 = pixConvert2To8(pix, (byte) 0, (byte) 85, (byte) 170, (byte) 255, 0);
break;
case 4:
pix2 = pixConvert4To8(pix, 0);
break;
case 8:
pix2 = pix;
break;
case 24:
pix2 = pix;
break;
default:
assert false;
}
tempPix = pix = pix2;
int channels = pix.d() / 8;
dtype = CV_8UC(channels);
Mat mat = new Mat(height, width, dtype, pix.data(), 4 * pix.wpl());
mat2 = new Mat(height, width, CV_8UC(channels));
// swap bytes if needed
int[] swap = {0, channels - 1, 1, channels - 2, 2, channels - 3, 3, channels - 4},
copy = {0, 0, 1, 1, 2, 2, 3, 3},
fromTo = channels > 1 && ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN) ? swap : copy;
mixChannels(mat, 1, mat2, 1, fromTo, Math.min(channels, fromTo.length / 2));
} else if (pix.d() == 16){
dtype = CV_16UC(pix.d() / 16);
} else if (pix.d() == 32) {
dtype = CV_32FC(pix.d() / 32);
}
mat2 = new Mat(height, width, dtype, pix.data());
if (tempPix != null) {
pixDestroy(tempPix);
}
return mat2;
}
public INDArray asMatrix(String filename) throws IOException {
return asMatrix(new File(filename));
}
@Override
public INDArray asMatrix(File f) throws IOException {
return asMatrix(f, true);
}
@Override
public INDArray asMatrix(File f, boolean nchw) throws IOException {
try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f))) {
return asMatrix(bis, nchw);
}
}
@Override
public INDArray asMatrix(InputStream is) throws IOException {
return asMatrix(is, true);
}
@Override
public INDArray asMatrix(InputStream inputStream, boolean nchw) throws IOException {
Mat mat = streamToMat(inputStream);
INDArray a;
if (this.multiPageMode != null) {
a = asMatrix(mat.data(), mat.cols());
}else{
Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR);
if (image == null || image.empty()) {
PIX pix = pixReadMem(mat.data(), mat.cols());
if (pix == null) {
throw new IOException("Could not decode image from input stream");
}
image = convert(pix);
pixDestroy(pix);
}
a = asMatrix(image);
image.deallocate();
}
if(nchw) {
return a;
} else {
return a.permute(0, 2, 3, 1); //NCHW to NHWC
}
}
/**
* Read the stream to the buffer, and return the number of bytes read
* @param is Input stream to read
* @return Mat with the buffer data as a row vector
* @throws IOException
*/
private Mat streamToMat(InputStream is) throws IOException {
byte[] buffer = IOUtils.toByteArray(is);
Mat bufferMat = null;
if (buffer.length <= 0) {
throw new IOException("Could not decode image from input stream: input stream was empty (no data)");
}
bufferMat = new Mat(buffer);
return bufferMat;
}
public Image asImageMatrix(String filename) throws IOException {
return asImageMatrix(new File(filename));
}
@Override
public Image asImageMatrix(File f) throws IOException {
return asImageMatrix(f, true);
}
@Override
public Image asImageMatrix(File f, boolean nchw) throws IOException {
try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f))) {
return asImageMatrix(bis, nchw);
}
}
@Override
public Image asImageMatrix(InputStream is) throws IOException {
return asImageMatrix(is, true);
}
@Override
public Image asImageMatrix(InputStream inputStream, boolean nchw) throws IOException {
Mat mat = streamToMat(inputStream);
Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR);
if (image == null || image.empty()) {
PIX pix = pixReadMem(mat.data(), mat.cols());
if (pix == null) {
throw new IOException("Could not decode image from input stream");
}
image = convert(pix);
pixDestroy(pix);
}
INDArray a = asMatrix(image);
if(!nchw)
a = a.permute(0,2,3,1); //NCHW to NHWC
Image i = new Image(a, image.channels(), image.rows(), image.cols());
image.deallocate();
return i;
}
/**
* Calls {@link AndroidNativeImageLoader#asMatrix(android.graphics.Bitmap)} or
* {@link Java2DNativeImageLoader#asMatrix(java.awt.image.BufferedImage)}.
* @param image as a {@link android.graphics.Bitmap} or {@link java.awt.image.BufferedImage}
* @return the matrix or null for unsupported object classes
* @throws IOException
*/
public INDArray asMatrix(Object image) throws IOException {
INDArray array = null;
if (array == null) {
try {
array = new AndroidNativeImageLoader(this).asMatrix(image);
} catch (NoClassDefFoundError e) {
// ignore
}
}
if (array == null) {
try {
array = new Java2DNativeImageLoader(this).asMatrix(image);
} catch (NoClassDefFoundError e) {
// ignore
}
}
return array;
}
protected void fillNDArray(Mat image, INDArray ret) {
long rows = image.rows();
long cols = image.cols();
long channels = image.channels();
if (ret.length() != rows * cols * channels) {
throw new ND4JIllegalStateException("INDArray provided to store image not equal to image: {channels: "
+ channels + ", rows: " + rows + ", columns: " + cols + "}");
}
Indexer idx = image.createIndexer(direct);
Pointer pointer = ret.data().pointer();
long[] stride = ret.stride();
boolean done = false;
PagedPointer pagedPointer = new PagedPointer(pointer, rows * cols * channels,
ret.offset() * Nd4j.sizeOfDataType(ret.data().dataType()));
if (pointer instanceof FloatPointer) {
FloatIndexer retidx = FloatIndexer.create(pagedPointer.asFloatPointer(),
new long[] {channels, rows, cols}, new long[] {stride[0], stride[1], stride[2]}, direct);
if (idx instanceof UByteIndexer) {
UByteIndexer ubyteidx = (UByteIndexer) idx;
for (long k = 0; k < channels; k++) {
for (long i = 0; i < rows; i++) {
for (long j = 0; j < cols; j++) {
retidx.put(k, i, j, ubyteidx.get(i, j, k));
}
}
}
done = true;
} else if (idx instanceof UShortIndexer) {
UShortIndexer ushortidx = (UShortIndexer) idx;
for (long k = 0; k < channels; k++) {
for (long i = 0; i < rows; i++) {
for (long j = 0; j < cols; j++) {
retidx.put(k, i, j, ushortidx.get(i, j, k));
}
}
}
done = true;
} else if (idx instanceof IntIndexer) {
IntIndexer intidx = (IntIndexer) idx;
for (long k = 0; k < channels; k++) {
for (long i = 0; i < rows; i++) {
for (long j = 0; j < cols; j++) {
retidx.put(k, i, j, intidx.get(i, j, k));
}
}
}
done = true;
} else if (idx instanceof FloatIndexer) {
FloatIndexer floatidx = (FloatIndexer) idx;
for (long k = 0; k < channels; k++) {
for (long i = 0; i < rows; i++) {
for (long j = 0; j < cols; j++) {
retidx.put(k, i, j, floatidx.get(i, j, k));
}
}
}
done = true;
}
retidx.release();
} else if (pointer instanceof DoublePointer) {
DoubleIndexer retidx = DoubleIndexer.create((DoublePointer) pagedPointer.asDoublePointer(),
new long[] {channels, rows, cols}, new long[] {stride[0], stride[1], stride[2]}, direct);
if (idx instanceof UByteIndexer) {
UByteIndexer ubyteidx = (UByteIndexer) idx;
for (long k = 0; k < channels; k++) {
for (long i = 0; i < rows; i++) {
for (long j = 0; j < cols; j++) {
retidx.put(k, i, j, ubyteidx.get(i, j, k));
}
}
}
done = true;
} else if (idx instanceof UShortIndexer) {
UShortIndexer ushortidx = (UShortIndexer) idx;
for (long k = 0; k < channels; k++) {
for (long i = 0; i < rows; i++) {
for (long j = 0; j < cols; j++) {
retidx.put(k, i, j, ushortidx.get(i, j, k));
}
}
}
done = true;
} else if (idx instanceof IntIndexer) {
IntIndexer intidx = (IntIndexer) idx;
for (long k = 0; k < channels; k++) {
for (long i = 0; i < rows; i++) {
for (long j = 0; j < cols; j++) {
retidx.put(k, i, j, intidx.get(i, j, k));
}
}
}
done = true;
} else if (idx instanceof FloatIndexer) {
FloatIndexer floatidx = (FloatIndexer) idx;
for (long k = 0; k < channels; k++) {
for (long i = 0; i < rows; i++) {
for (long j = 0; j < cols; j++) {
retidx.put(k, i, j, floatidx.get(i, j, k));
}
}
}
done = true;
}
retidx.release();
}
if (!done) {
for (long k = 0; k < channels; k++) {
for (long i = 0; i < rows; i++) {
for (long j = 0; j < cols; j++) {
if (ret.rank() == 3) {
ret.putScalar(k, i, j, idx.getDouble(i, j, k));
} else if (ret.rank() == 4) {
ret.putScalar(1, k, i, j, idx.getDouble(i, j, k));
} else if (ret.rank() == 2) {
ret.putScalar(i, j, idx.getDouble(i, j));
} else
throw new ND4JIllegalStateException("NativeImageLoader expects 2D, 3D or 4D output array, but " + ret.rank() + "D array was given");
}
}
}
}
idx.release();
image.data();
Nd4j.getAffinityManager().tagLocation(ret, AffinityManager.Location.HOST);
}
public void asMatrixView(InputStream is, INDArray view) throws IOException {
Mat mat = streamToMat(is);
Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR);
if (image == null || image.empty()) {
PIX pix = pixReadMem(mat.data(), mat.cols());
if (pix == null) {
throw new IOException("Could not decode image from input stream");
}
image = convert(pix);
pixDestroy(pix);
}
if (image == null)
throw new RuntimeException();
asMatrixView(image, view);
image.deallocate();
}
public void asMatrixView(String filename, INDArray view) throws IOException {
asMatrixView(new File(filename), view);
}
public void asMatrixView(File f, INDArray view) throws IOException {
try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f))) {
asMatrixView(bis, view);
}
}
public void asMatrixView(Mat image, INDArray view) throws IOException {
transformImage(image, view);
}
public void asMatrixView(org.opencv.core.Mat image, INDArray view) throws IOException {
transformImage(image, view);
}
public INDArray asMatrix(Frame image) throws IOException {
return asMatrix(converter.convert(image));
}
public INDArray asMatrix(org.opencv.core.Mat image) throws IOException {
INDArray ret = transformImage(image, null);
return ret.reshape(ArrayUtil.combine(new long[] {1}, ret.shape()));
}
public INDArray asMatrix(Mat image) throws IOException {
INDArray ret = transformImage(image, null);
return ret.reshape(ArrayUtil.combine(new long[] {1}, ret.shape()));
}
protected INDArray transformImage(org.opencv.core.Mat image, INDArray ret) throws IOException {
Frame f = converter.convert(image);
return transformImage(converter.convert(f), ret);
}
protected INDArray transformImage(Mat image, INDArray ret) throws IOException {
if (imageTransform != null && converter != null) {
ImageWritable writable = new ImageWritable(converter.convert(image));
writable = imageTransform.transform(writable);
image = converter.convert(writable.getFrame());
}
Mat image2 = null, image3 = null, image4 = null;
if (channels > 0 && image.channels() != channels) {
int code = -1;
switch (image.channels()) {
case 1:
switch ((int)channels) {
case 3:
code = CV_GRAY2BGR;
break;
case 4:
code = CV_GRAY2RGBA;
break;
}
break;
case 3:
switch ((int)channels) {
case 1:
code = CV_BGR2GRAY;
break;
case 4:
code = CV_BGR2RGBA;
break;
}
break;
case 4:
switch ((int)channels) {
case 1:
code = CV_RGBA2GRAY;
break;
case 3:
code = CV_RGBA2BGR;
break;
}
break;
}
if (code < 0) {
throw new IOException("Cannot convert from " + image.channels() + " to " + channels + " channels.");
}
image2 = new Mat();
cvtColor(image, image2, code);
image = image2;
}
if (centerCropIfNeeded) {
image3 = centerCropIfNeeded(image);
if (image3 != image) {
image = image3;
} else {
image3 = null;
}
}
image4 = scalingIfNeed(image);
if (image4 != image) {
image = image4;
} else {
image4 = null;
}
if (ret == null) {
int rows = image.rows();
int cols = image.cols();
int channels = image.channels();
ret = Nd4j.create(channels, rows, cols);
}
fillNDArray(image, ret);
image.data(); // dummy call to make sure it does not get deallocated prematurely
if (image2 != null) {
image2.deallocate();
}
if (image3 != null) {
image3.deallocate();
}
if (image4 != null) {
image4.deallocate();
}
return ret;
}
// TODO build flexibility on where to crop the image
protected Mat centerCropIfNeeded(Mat img) {
int x = 0;
int y = 0;
int height = img.rows();
int width = img.cols();
int diff = Math.abs(width - height) / 2;
if (width > height) {
x = diff;
width = width - diff;
} else if (height > width) {
y = diff;
height = height - diff;
}
return img.apply(new Rect(x, y, width, height));
}
protected Mat scalingIfNeed(Mat image) {
return scalingIfNeed(image, height, width);
}
protected Mat scalingIfNeed(Mat image, long dstHeight, long dstWidth) {
Mat scaled = image;
if (dstHeight > 0 && dstWidth > 0 && (image.rows() != dstHeight || image.cols() != dstWidth)) {
resize(image, scaled = new Mat(), new Size(
(int)Math.min(dstWidth, Integer.MAX_VALUE),
(int)Math.min(dstHeight, Integer.MAX_VALUE)));
}
return scaled;
}
public ImageWritable asWritable(String filename) throws IOException {
return asWritable(new File(filename));
}
/**
* Convert a file to a INDArray
*
* @param f the image to convert
* @return INDArray
* @throws IOException
*/
public ImageWritable asWritable(File f) throws IOException {
try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f))) {
Mat mat = streamToMat(bis);
Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR);
if (image == null || image.empty()) {
PIX pix = pixReadMem(mat.data(), mat.cols());
if (pix == null) {
throw new IOException("Could not decode image from input stream");
}
image = convert(pix);
pixDestroy(pix);
}
ImageWritable writable = new ImageWritable(converter.convert(image));
return writable;
}
}
/**
* Convert ImageWritable to INDArray
*
* @param writable ImageWritable to convert
* @return INDArray
* @throws IOException
*/
public INDArray asMatrix(ImageWritable writable) throws IOException {
Mat image = converter.convert(writable.getFrame());
return asMatrix(image);
}
/** Returns {@code asFrame(array, -1)}. */
public Frame asFrame(INDArray array) {
return converter.convert(asMat(array));
}
/**
* Converts an INDArray to a JavaCV Frame. Only intended for images with rank 3.
*
* @param array to convert
* @param dataType from JavaCV (DEPTH_FLOAT, DEPTH_UBYTE, etc), or -1 to use same type as the INDArray
* @return data copied to a Frame
*/
public Frame asFrame(INDArray array, int dataType) {
return converter.convert(asMat(array, OpenCVFrameConverter.getMatDepth(dataType)));
}
/** Returns {@code asMat(array, -1)}. */
public Mat asMat(INDArray array) {
return asMat(array, -1);
}
/**
* Converts an INDArray to an OpenCV Mat. Only intended for images with rank 3.
*
* @param array to convert
* @param dataType from OpenCV (CV_32F, CV_8U, etc), or -1 to use same type as the INDArray
* @return data copied to a Mat
*/
public Mat asMat(INDArray array, int dataType) {
if (array.rank() > 4 || (array.rank() > 3 && array.size(0) != 1)) {
throw new UnsupportedOperationException("Only rank 3 (or rank 4 with size(0) == 1) arrays supported");
}
int rank = array.rank();
long[] stride = array.stride();
long offset = array.offset();
Pointer pointer = array.data().pointer().position(offset);
long rows = array.size(rank == 3 ? 1 : 2);
long cols = array.size(rank == 3 ? 2 : 3);
long channels = array.size(rank == 3 ? 0 : 1);
boolean done = false;
if (dataType < 0) {
dataType = pointer instanceof DoublePointer ? CV_64F : CV_32F;
}
Mat mat = new Mat((int)Math.min(rows, Integer.MAX_VALUE), (int)Math.min(cols, Integer.MAX_VALUE),
CV_MAKETYPE(dataType, (int)Math.min(channels, Integer.MAX_VALUE)));
Indexer matidx = mat.createIndexer(direct);
Nd4j.getAffinityManager().ensureLocation(array, AffinityManager.Location.HOST);
if (pointer instanceof FloatPointer && dataType == CV_32F) {
FloatIndexer ptridx = FloatIndexer.create((FloatPointer)pointer, new long[] {channels, rows, cols},
new long[] {stride[rank == 3 ? 0 : 1], stride[rank == 3 ? 1 : 2], stride[rank == 3 ? 2 : 3]}, direct);
FloatIndexer idx = (FloatIndexer)matidx;
for (long k = 0; k < channels; k++) {
for (long i = 0; i < rows; i++) {
for (long j = 0; j < cols; j++) {
idx.put(i, j, k, ptridx.get(k, i, j));
}
}
}
done = true;
ptridx.release();
} else if (pointer instanceof DoublePointer && dataType == CV_64F) {
DoubleIndexer ptridx = DoubleIndexer.create((DoublePointer)pointer, new long[] {channels, rows, cols},
new long[] {stride[rank == 3 ? 0 : 1], stride[rank == 3 ? 1 : 2], stride[rank == 3 ? 2 : 3]}, direct);
DoubleIndexer idx = (DoubleIndexer)matidx;
for (long k = 0; k < channels; k++) {
for (long i = 0; i < rows; i++) {
for (long j = 0; j < cols; j++) {
idx.put(i, j, k, ptridx.get(k, i, j));
}
}
}
done = true;
ptridx.release();
}
if (!done) {
for (long k = 0; k < channels; k++) {
for (long i = 0; i < rows; i++) {
for (long j = 0; j < cols; j++) {
if (rank == 3) {
matidx.putDouble(new long[] {i, j, k}, array.getDouble(k, i, j));
} else {
matidx.putDouble(new long[] {i, j, k}, array.getDouble(0, k, i, j));
}
}
}
}
}
matidx.release();
return mat;
}
/**
* Read multipage tiff and load into INDArray
*
* @param bytes
* @return INDArray
* @throws IOException
*/
private INDArray asMatrix(BytePointer bytes, long length) throws IOException {
PIXA pixa;
pixa = pixaReadMemMultipageTiff(bytes, length);
INDArray data;
INDArray currentD;
INDArrayIndex[] index = null;
switch (this.multiPageMode) {
case MINIBATCH:
data = Nd4j.create(pixa.n(), 1, 1, pixa.pix(0).h(), pixa.pix(0).w());
break;
case FIRST:
data = Nd4j.create(1, 1, 1, pixa.pix(0).h(), pixa.pix(0).w());
PIX pix = pixa.pix(0);
currentD = asMatrix(convert(pix));
pixDestroy(pix);
index = new INDArrayIndex[]{NDArrayIndex.point(0), NDArrayIndex.point(0), NDArrayIndex.point(0),
NDArrayIndex.all(), NDArrayIndex.all()};
data.put(index , currentD.get(NDArrayIndex.all(), NDArrayIndex.all(),
NDArrayIndex.all(), NDArrayIndex.all()));
return data;
default: throw new UnsupportedOperationException("Unsupported MultiPageMode: " + multiPageMode);
}
for (int i = 0; i < pixa.n(); i++) {
PIX pix = pixa.pix(i);
currentD = asMatrix(convert(pix));
pixDestroy(pix);
switch (this.multiPageMode) {
case MINIBATCH:
index = new INDArrayIndex[]{NDArrayIndex.point(i),NDArrayIndex.all(), NDArrayIndex.all(),NDArrayIndex.all(),NDArrayIndex.all()};
break;
//
default: throw new UnsupportedOperationException("Unsupported MultiPageMode: " + multiPageMode);
}
data.put(index , currentD.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.all(),NDArrayIndex.all()));
}
return data;
}
}
@@ -0,0 +1,536 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.recordreader;
import org.nd4j.shade.guava.base.Preconditions;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.datavec.api.conf.Configuration;
import org.datavec.api.io.labels.PathLabelGenerator;
import org.datavec.api.io.labels.PathMultiLabelGenerator;
import org.datavec.api.records.Record;
import org.datavec.api.records.metadata.RecordMetaData;
import org.datavec.api.records.metadata.RecordMetaDataURI;
import org.datavec.api.records.reader.BaseRecordReader;
import org.datavec.api.split.FileSplit;
import org.datavec.api.split.InputSplit;
import org.datavec.api.split.InputStreamInputSplit;
import org.datavec.api.util.files.FileFromPathIterator;
import org.datavec.api.util.files.URIUtil;
import org.datavec.api.util.ndarray.RecordConverter;
import org.datavec.api.writable.IntWritable;
import org.datavec.api.writable.NDArrayWritable;
import org.datavec.api.writable.Writable;
import org.datavec.api.writable.batch.NDArrayRecordBatch;
import org.datavec.image.loader.BaseImageLoader;
import org.datavec.image.loader.ImageLoader;
import org.datavec.image.loader.NativeImageLoader;
import org.datavec.image.transform.ImageTransform;
import org.nd4j.linalg.api.concurrency.AffinityManager;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.factory.Nd4j;
import java.io.*;
import java.net.URI;
import java.util.*;
@Slf4j
public abstract class BaseImageRecordReader extends BaseRecordReader {
protected boolean finishedInputStreamSplit;
protected Iterator<File> iter;
protected Configuration conf;
protected File currentFile;
protected PathLabelGenerator labelGenerator = null;
protected PathMultiLabelGenerator labelMultiGenerator = null;
protected List<String> labels = new ArrayList<>();
protected boolean appendLabel = false;
protected boolean writeLabel = false;
protected List<Writable> record;
protected boolean hitImage = false;
protected long height = 28, width = 28, channels = 1;
protected boolean cropImage = false;
protected ImageTransform imageTransform;
protected BaseImageLoader imageLoader;
protected InputSplit inputSplit;
protected Map<String, String> fileNameMap = new LinkedHashMap<>();
protected String pattern; // Pattern to split and segment file name, pass in regex
protected int patternPosition = 0;
@Getter @Setter
protected boolean logLabelCountOnInit = true;
@Getter @Setter
protected boolean nchw_channels_first = true;
public final static String HEIGHT = NAME_SPACE + ".height";
public final static String WIDTH = NAME_SPACE + ".width";
public final static String CHANNELS = NAME_SPACE + ".channels";
public final static String CROP_IMAGE = NAME_SPACE + ".cropimage";
public final static String IMAGE_LOADER = NAME_SPACE + ".imageloader";
public BaseImageRecordReader() {}
public BaseImageRecordReader(long height, long width, long channels, PathLabelGenerator labelGenerator) {
this(height, width, channels, labelGenerator, null);
}
public BaseImageRecordReader(long height, long width, long channels, PathMultiLabelGenerator labelGenerator) {
this(height, width, channels, null, labelGenerator,null);
}
public BaseImageRecordReader(long height, long width, long channels, PathLabelGenerator labelGenerator,
ImageTransform imageTransform) {
this(height, width, channels, labelGenerator, null, imageTransform);
}
protected BaseImageRecordReader(long height, long width, long channels, PathLabelGenerator labelGenerator,
PathMultiLabelGenerator labelMultiGenerator, ImageTransform imageTransform) {
this(height, width, channels, true, labelGenerator, labelMultiGenerator, imageTransform);
}
protected BaseImageRecordReader(long height, long width, long channels, boolean nchw_channels_first, PathLabelGenerator labelGenerator,
PathMultiLabelGenerator labelMultiGenerator, ImageTransform imageTransform) {
this.height = height;
this.width = width;
this.channels = channels;
this.labelGenerator = labelGenerator;
this.labelMultiGenerator = labelMultiGenerator;
this.imageTransform = imageTransform;
this.appendLabel = (labelGenerator != null || labelMultiGenerator != null);
this.nchw_channels_first = nchw_channels_first;
}
protected boolean containsFormat(String format) {
for (String format2 : imageLoader.getAllowedFormats())
if (format.endsWith("." + format2))
return true;
return false;
}
@Override
public void initialize(InputSplit split) throws IOException {
if (imageLoader == null) {
imageLoader = new NativeImageLoader(height, width, channels, imageTransform);
}
if(split instanceof InputStreamInputSplit) {
this.inputSplit = split;
this.finishedInputStreamSplit = false;
return;
}
inputSplit = split;
URI[] locations = split.locations();
if (locations != null && locations.length >= 1) {
if (appendLabel && labelGenerator != null && labelGenerator.inferLabelClasses()) {
Set<String> labelsSet = new HashSet<>();
for (URI location : locations) {
File imgFile = new File(location);
String name = labelGenerator.getLabelForPath(location).toString();
labelsSet.add(name);
if (pattern != null) {
String label = name.split(pattern)[patternPosition];
fileNameMap.put(imgFile.toString(), label);
}
}
labels.clear();
labels.addAll(labelsSet);
if(logLabelCountOnInit) {
log.info("ImageRecordReader: {} label classes inferred using label generator {}", labelsSet.size(), labelGenerator.getClass().getSimpleName());
}
}
iter = new FileFromPathIterator(inputSplit.locationsPathIterator()); //This handles randomization internally if necessary
} else
throw new IllegalArgumentException("No path locations found in the split.");
if (split instanceof FileSplit) {
//remove the root directory
FileSplit split1 = (FileSplit) split;
labels.remove(split1.getRootDir());
}
//To ensure consistent order for label assignment (irrespective of file iteration order), we want to sort the list of labels
Collections.sort(labels);
}
@Override
public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
this.appendLabel = conf.getBoolean(APPEND_LABEL, appendLabel);
this.labels = new ArrayList<>(conf.getStringCollection(LABELS));
this.height = conf.getLong(HEIGHT, height);
this.width = conf.getLong(WIDTH, width);
this.channels = conf.getLong(CHANNELS, channels);
this.cropImage = conf.getBoolean(CROP_IMAGE, cropImage);
if ("imageio".equals(conf.get(IMAGE_LOADER))) {
this.imageLoader = new ImageLoader(height, width, channels, cropImage);
} else {
this.imageLoader = new NativeImageLoader(height, width, channels, imageTransform);
}
this.conf = conf;
initialize(split);
}
/**
* Called once at initialization.
*
* @param split the split that defines the range of records to read
* @param imageTransform the image transform to use to transform images while loading them
* @throws java.io.IOException
*/
public void initialize(InputSplit split, ImageTransform imageTransform) throws IOException {
this.imageLoader = null;
this.imageTransform = imageTransform;
initialize(split);
}
/**
* Called once at initialization.
*
* @param conf a configuration for initialization
* @param split the split that defines the range of records to read
* @param imageTransform the image transform to use to transform images while loading them
* @throws java.io.IOException
* @throws InterruptedException
*/
public void initialize(Configuration conf, InputSplit split, ImageTransform imageTransform)
throws IOException, InterruptedException {
this.imageLoader = null;
this.imageTransform = imageTransform;
initialize(conf, split);
}
@Override
public List<Writable> next() {
if(inputSplit instanceof InputStreamInputSplit) {
InputStreamInputSplit inputStreamInputSplit = (InputStreamInputSplit) inputSplit;
try {
NDArrayWritable ndArrayWritable = new NDArrayWritable(imageLoader.asMatrix(inputStreamInputSplit.getIs()));
finishedInputStreamSplit = true;
return Arrays.<Writable>asList(ndArrayWritable);
} catch (IOException e) {
log.error("",e);
}
}
if (iter != null) {
List<Writable> ret;
File image = iter.next();
currentFile = image;
if (image.isDirectory())
return next();
try {
invokeListeners(image);
INDArray array = imageLoader.asMatrix(image);
if(!nchw_channels_first){
array = array.permute(0,2,3,1); //NCHW to NHWC
}
Nd4j.getAffinityManager().ensureLocation(array, AffinityManager.Location.DEVICE);
ret = RecordConverter.toRecord(array);
if (appendLabel || writeLabel){
if(labelMultiGenerator != null){
ret.addAll(labelMultiGenerator.getLabels(image.getPath()));
} else {
if (labelGenerator.inferLabelClasses()) {
//Standard classification use case (i.e., handle String -> integer conversion
ret.add(new IntWritable(labels.indexOf(getLabel(image.getPath()))));
} else {
//Regression use cases, and PathLabelGenerator instances that already map to integers
ret.add(labelGenerator.getLabelForPath(image.getPath()));
}
}
}
} catch (Exception e) {
throw new RuntimeException(e);
}
return ret;
} else if (record != null) {
hitImage = true;
invokeListeners(record);
return record;
}
throw new IllegalStateException("No more elements");
}
@Override
public boolean hasNext() {
if(inputSplit instanceof InputStreamInputSplit) {
return finishedInputStreamSplit;
}
if (iter != null) {
return iter.hasNext();
} else if (record != null) {
return !hitImage;
}
throw new IllegalStateException("Indeterminant state: record must not be null, or a file iterator must exist");
}
@Override
public boolean batchesSupported() {
return (imageLoader instanceof NativeImageLoader);
}
@Override
public List<List<Writable>> next(int num) {
Preconditions.checkArgument(num > 0, "Number of examples must be > 0: got %s", num);
if (imageLoader == null) {
imageLoader = new NativeImageLoader(height, width, channels, imageTransform);
}
List<File> currBatch = new ArrayList<>();
int cnt = 0;
int numCategories = (appendLabel || writeLabel) ? labels.size() : 0;
List<Integer> currLabels = null;
List<Writable> currLabelsWritable = null;
List<List<Writable>> multiGenLabels = null;
while (cnt < num && iter.hasNext()) {
currentFile = iter.next();
currBatch.add(currentFile);
invokeListeners(currentFile);
if (appendLabel || writeLabel) {
//Collect the label Writables from the label generators
if(labelMultiGenerator != null){
if(multiGenLabels == null)
multiGenLabels = new ArrayList<>();
multiGenLabels.add(labelMultiGenerator.getLabels(currentFile.getPath()));
} else {
if (labelGenerator.inferLabelClasses()) {
if (currLabels == null)
currLabels = new ArrayList<>();
currLabels.add(labels.indexOf(getLabel(currentFile.getPath())));
} else {
if (currLabelsWritable == null)
currLabelsWritable = new ArrayList<>();
currLabelsWritable.add(labelGenerator.getLabelForPath(currentFile.getPath()));
}
}
}
cnt++;
}
INDArray features = Nd4j.createUninitialized(new long[] {cnt, channels, height, width}, 'c');
Nd4j.getAffinityManager().tagLocation(features, AffinityManager.Location.HOST);
for (int i = 0; i < cnt; i++) {
try {
((NativeImageLoader) imageLoader).asMatrixView(currBatch.get(i),
features.tensorAlongDimension(i, 1, 2, 3));
} catch (Exception e) {
System.out.println("Image file failed during load: " + currBatch.get(i).getAbsolutePath());
throw new RuntimeException(e);
}
}
if(!nchw_channels_first){
features = features.permute(0,2,3,1); //NCHW to NHWC
}
Nd4j.getAffinityManager().ensureLocation(features, AffinityManager.Location.DEVICE);
List<INDArray> ret = new ArrayList<>();
ret.add(features);
if (appendLabel || writeLabel) {
//And convert the previously collected label Writables from the label generators
if(labelMultiGenerator != null){
List<Writable> temp = new ArrayList<>();
List<Writable> first = multiGenLabels.get(0);
for(int col=0; col<first.size(); col++ ){
temp.clear();
for (List<Writable> multiGenLabel : multiGenLabels) {
temp.add(multiGenLabel.get(col));
}
INDArray currCol = RecordConverter.toMinibatchArray(temp);
ret.add(currCol);
}
} else {
INDArray labels;
if (labelGenerator.inferLabelClasses()) {
//Standard classification use case (i.e., handle String -> integer conversion)
labels = Nd4j.create(cnt, numCategories, 'c');
Nd4j.getAffinityManager().tagLocation(labels, AffinityManager.Location.HOST);
for (int i = 0; i < currLabels.size(); i++) {
labels.putScalar(i, currLabels.get(i), 1.0f);
}
} else {
//Regression use cases, and PathLabelGenerator instances that already map to integers
if (currLabelsWritable.get(0) instanceof NDArrayWritable) {
List<INDArray> arr = new ArrayList<>();
for (Writable w : currLabelsWritable) {
arr.add(((NDArrayWritable) w).get());
}
labels = Nd4j.concat(0, arr.toArray(new INDArray[arr.size()]));
} else {
labels = RecordConverter.toMinibatchArray(currLabelsWritable);
}
}
ret.add(labels);
}
}
return new NDArrayRecordBatch(ret);
}
@Override
public void close() throws IOException {
//No op
}
@Override
public void setConf(Configuration conf) {
this.conf = conf;
}
@Override
public Configuration getConf() {
return conf;
}
/**
* Get the label from the given path
*
* @param path the path to get the label from
* @return the label for the given path
*/
public String getLabel(String path) {
if (labelGenerator != null) {
return labelGenerator.getLabelForPath(path).toString();
}
if (fileNameMap != null && fileNameMap.containsKey(path))
return fileNameMap.get(path);
return (new File(path)).getParentFile().getName();
}
/**
* Accumulate the label from the path
*
* @param path the path to get the label from
*/
protected void accumulateLabel(String path) {
String name = getLabel(path);
if (!labels.contains(name))
labels.add(name);
}
/**
* Returns the file loaded last by {@link #next()}.
*/
public File getCurrentFile() {
return currentFile;
}
/**
* Sets manually the file returned by {@link #getCurrentFile()}.
*/
public void setCurrentFile(File currentFile) {
this.currentFile = currentFile;
}
@Override
public List<String> getLabels() {
return labels;
}
public void setLabels(List<String> labels) {
this.labels = labels;
this.writeLabel = true;
}
@Override
public void reset() {
if (inputSplit == null)
throw new UnsupportedOperationException("Cannot reset without first initializing");
inputSplit.reset();
if (iter != null) {
iter = new FileFromPathIterator(inputSplit.locationsPathIterator());
} else if (record != null) {
hitImage = false;
}
}
@Override
public boolean resetSupported(){
if(inputSplit == null){
return false;
}
return inputSplit.resetSupported();
}
/**
* Returns {@code getLabels().size()}.
*/
public int numLabels() {
return labels.size();
}
@Override
public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
invokeListeners(uri);
if (imageLoader == null) {
imageLoader = new NativeImageLoader(height, width, channels, imageTransform);
}
INDArray array = imageLoader.asMatrix(dataInputStream);
if(!nchw_channels_first)
array = array.permute(0,2,3,1);
List<Writable> ret = RecordConverter.toRecord(array);
if (appendLabel)
ret.add(new IntWritable(labels.indexOf(getLabel(uri.getPath()))));
return ret;
}
@Override
public Record nextRecord() {
List<Writable> list = next();
URI uri = URIUtil.fileToURI(currentFile);
return new org.datavec.api.records.impl.Record(list, new RecordMetaDataURI(uri, BaseImageRecordReader.class));
}
@Override
public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException {
return loadFromMetaData(Collections.singletonList(recordMetaData)).get(0);
}
@Override
public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
List<Record> out = new ArrayList<>();
for (RecordMetaData meta : recordMetaDatas) {
URI uri = meta.getURI();
File f = new File(uri);
List<Writable> next;
try (DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(f)))) {
next = record(uri, dis);
}
out.add(new org.datavec.api.records.impl.Record(next, meta));
}
return out;
}
}
@@ -0,0 +1,97 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.recordreader;
import org.datavec.api.io.labels.PathLabelGenerator;
import org.datavec.api.io.labels.PathMultiLabelGenerator;
import org.datavec.image.transform.ImageTransform;
public class ImageRecordReader extends BaseImageRecordReader {
/** Loads images with height = 28, width = 28, and channels = 1, appending no labels.
* Output format is NCHW (channels first) - [numExamples, 1, 28, 28]*/
public ImageRecordReader() {
super();
}
/** Loads images with given height, width, and channels, appending labels returned by the generator.
* Output format is NCHW (channels first) - [numExamples, channels, height, width]
*/
public ImageRecordReader(long height, long width, long channels, PathLabelGenerator labelGenerator) {
super(height, width, channels, labelGenerator);
}
/** Loads images with given height, width, and channels, appending labels returned by the generator.
* Output format is NCHW (channels first) - [numExamples, channels, height, width]
*/
public ImageRecordReader(long height, long width, long channels, PathMultiLabelGenerator labelGenerator) {
super(height, width, channels, labelGenerator);
}
/** Loads images with given height, width, and channels, appending no labels - in NCHW (channels first) format */
public ImageRecordReader(long height, long width, long channels) {
super(height, width, channels, (PathLabelGenerator) null);
}
/** Loads images with given height, width, and channels, appending no labels - in specified format<br>
* If {@code nchw_channels_first == true} output format is NCHW (channels first) - [numExamples, channels, height, width]<br>
* If {@code nchw_channels_first == false} output format is NHWC (channels last) - [numExamples, height, width, channels]<br>
*/
public ImageRecordReader(long height, long width, long channels, boolean nchw_channels_first) {
super(height, width, channels, nchw_channels_first, null, null, null);
}
/** Loads images with given height, width, and channels, appending labels returned by the generator.
* Output format is NCHW (channels first) - [numExamples, channels, height, width] */
public ImageRecordReader(long height, long width, long channels, PathLabelGenerator labelGenerator,
ImageTransform imageTransform) {
super(height, width, channels, labelGenerator, imageTransform);
}
/** Loads images with given height, width, and channels, appending labels returned by the generator.<br>
* If {@code nchw_channels_first == true} output format is NCHW (channels first) - [numExamples, channels, height, width]<br>
* If {@code nchw_channels_first == false} output format is NHWC (channels last) - [numExamples, height, width, channels]<br>
*/
public ImageRecordReader(long height, long width, long channels, boolean nchw_channels_first, PathLabelGenerator labelGenerator,
ImageTransform imageTransform) {
super(height, width, channels, nchw_channels_first, labelGenerator, null, imageTransform);
}
/** Loads images with given height, width, and channels, appending no labels.
* Output format is NCHW (channels first) - [numExamples, channels, height, width]*/
public ImageRecordReader(long height, long width, long channels, ImageTransform imageTransform) {
super(height, width, channels, null, imageTransform);
}
/** Loads images with given height, width, and channels, appending labels returned by the generator
* Output format is NCHW (channels first) - [numExamples, channels, height, width]*/
public ImageRecordReader(long height, long width, PathLabelGenerator labelGenerator) {
super(height, width, 1, labelGenerator);
}
/** Loads images with given height, width, and channels = 1, appending no labels.
* Output format is NCHW (channels first) - [numExamples, channels, height, width]*/
public ImageRecordReader(long height, long width) {
super(height, width, 1, null, null);
}
}
@@ -0,0 +1,54 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.recordreader.objdetect;
import lombok.Data;
@Data
public class ImageObject {
private final int x1;
private final int y1;
private final int x2;
private final int y2;
private final String label;
public ImageObject(int x1, int y1, int x2, int y2, String label){
if(x1 > x2 || y1 > y2){
throw new IllegalArgumentException("Invalid input: (x1,y1), top left position must have values less than" +
" (x2,y2) bottom right position. Got: (" + x1 + "," + y1 + "), (" + x2 + "," + y2 + ")");
}
this.x1 = x1;
this.y1 = y1;
this.x2 = x2;
this.y2 = y2;
this.label = label;
}
public double getXCenterPixels(){
return (x1 + x2) / 2.0;
}
public double getYCenterPixels(){
return (y1 + y2) / 2.0;
}
}
@@ -0,0 +1,32 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.recordreader.objdetect;
import java.net.URI;
import java.util.List;
public interface ImageObjectLabelProvider {
List<ImageObject> getImageObjectsForPath(String path);
List<ImageObject> getImageObjectsForPath(URI uri);
}
@@ -0,0 +1,306 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.recordreader.objdetect;
import org.datavec.api.split.FileSplit;
import org.datavec.api.split.InputSplit;
import org.datavec.api.util.files.FileFromPathIterator;
import org.datavec.api.writable.NDArrayWritable;
import org.datavec.api.writable.Writable;
import org.datavec.api.writable.batch.NDArrayRecordBatch;
import org.datavec.image.data.Image;
import org.datavec.image.loader.NativeImageLoader;
import org.datavec.image.recordreader.BaseImageRecordReader;
import org.datavec.image.util.ImageUtils;
import org.nd4j.common.base.Preconditions;
import org.nd4j.linalg.api.concurrency.AffinityManager;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.indexing.INDArrayIndex;
import org.datavec.api.records.Record;
import org.datavec.api.records.metadata.RecordMetaDataImageURI;
import org.datavec.api.util.files.URIUtil;
import org.datavec.api.util.ndarray.RecordConverter;
import org.datavec.image.transform.ImageTransform;
import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.*;
import static org.nd4j.linalg.indexing.NDArrayIndex.all;
import static org.nd4j.linalg.indexing.NDArrayIndex.point;
public class ObjectDetectionRecordReader extends BaseImageRecordReader {
private final int gridW;
private final int gridH;
private final ImageObjectLabelProvider labelProvider;
private final boolean nchw;
protected Image currentImage;
/**
* As per {@link #ObjectDetectionRecordReader(int, int, int, int, int, boolean, ImageObjectLabelProvider)} but hardcoded
* to NCHW format
*/
public ObjectDetectionRecordReader(int height, int width, int channels, int gridH, int gridW, ImageObjectLabelProvider labelProvider) {
this(height, width, channels, gridH, gridW, true, labelProvider);
}
/**
* Create ObjectDetectionRecordReader with
*
* @param height Height of the output images
* @param width Width of the output images
* @param channels Number of channels for the output images
* @param gridH Grid/quantization size (along height dimension) - Y axis
* @param gridW Grid/quantization size (along height dimension) - X axis
* @param nchw If true: return NCHW format labels with array shape [minibatch, 4+C, h, w]; if false, return
* NHWC format labels with array shape [minibatch, h, w, 4+C]
* @param labelProvider ImageObjectLabelProvider - used to look up which objects are in each image
*/
public ObjectDetectionRecordReader(int height, int width, int channels, int gridH, int gridW, boolean nchw, ImageObjectLabelProvider labelProvider) {
super(height, width, channels, null, null);
this.gridW = gridW;
this.gridH = gridH;
this.nchw = nchw;
this.labelProvider = labelProvider;
this.appendLabel = labelProvider != null;
}
/**
* As per {@link #ObjectDetectionRecordReader(int, int, int, int, int, boolean, ImageObjectLabelProvider, ImageTransform)}
* but hardcoded to NCHW format
*/
public ObjectDetectionRecordReader(int height, int width, int channels, int gridH, int gridW,
ImageObjectLabelProvider labelProvider, ImageTransform imageTransform) {
this(height, width, channels, gridH, gridW, true, labelProvider, imageTransform);
}
/**
* When imageTransform != null, object is removed if new center is outside of transformed image bounds.
*
* @param height Height of the output images
* @param width Width of the output images
* @param channels Number of channels for the output images
* @param gridH Grid/quantization size (along height dimension) - Y axis
* @param gridW Grid/quantization size (along height dimension) - X axis
* @param labelProvider ImageObjectLabelProvider - used to look up which objects are in each image
* @param nchw If true: return NCHW format labels with array shape [minibatch, 4+C, h, w]; if false, return
* NHWC format labels with array shape [minibatch, h, w, 4+C]
* @param imageTransform ImageTransform - used to transform image and coordinates
*/
public ObjectDetectionRecordReader(int height, int width, int channels, int gridH, int gridW, boolean nchw,
ImageObjectLabelProvider labelProvider, ImageTransform imageTransform) {
super(height, width, channels, null, null);
this.gridW = gridW;
this.gridH = gridH;
this.nchw = nchw;
this.labelProvider = labelProvider;
this.appendLabel = labelProvider != null;
this.imageTransform = imageTransform;
}
@Override
public List<Writable> next() {
return next(1).get(0);
}
@Override
public void initialize(InputSplit split) throws IOException {
if (imageLoader == null) {
imageLoader = new NativeImageLoader(height, width, channels, imageTransform);
}
inputSplit = split;
URI[] locations = split.locations();
Set<String> labelSet = new HashSet<>();
if (locations != null && locations.length >= 1) {
for (URI location : locations) {
List<ImageObject> imageObjects = labelProvider.getImageObjectsForPath(location);
for (ImageObject io : imageObjects) {
String name = io.getLabel();
if (!labelSet.contains(name)) {
labelSet.add(name);
}
}
}
iter = new FileFromPathIterator(inputSplit.locationsPathIterator()); //This handles randomization internally if necessary
} else {
throw new IllegalArgumentException("No path locations found in the split.");
}
if (split instanceof FileSplit) {
//remove the root directory
FileSplit split1 = (FileSplit) split;
labels.remove(split1.getRootDir());
}
//To ensure consistent order for label assignment (irrespective of file iteration order), we want to sort the list of labels
labels = new ArrayList<>(labelSet);
Collections.sort(labels);
}
@Override
public List<List<Writable>> next(int num) {
List<File> files = new ArrayList<>(num);
List<List<ImageObject>> objects = new ArrayList<>(num);
for (int i = 0; i < num && hasNext(); i++) {
File f = iter.next();
this.currentFile = f;
if (!f.isDirectory()) {
files.add(f);
objects.add(labelProvider.getImageObjectsForPath(f.getPath()));
}
}
int nClasses = labels.size();
INDArray outImg = Nd4j.create(files.size(), channels, height, width);
INDArray outLabel = Nd4j.create(files.size(), 4 + nClasses, gridH, gridW);
int exampleNum = 0;
for (int i = 0; i < files.size(); i++) {
File imageFile = files.get(i);
this.currentFile = imageFile;
try {
this.invokeListeners(imageFile);
Image image = this.imageLoader.asImageMatrix(imageFile);
this.currentImage = image;
Nd4j.getAffinityManager().ensureLocation(image.getImage(), AffinityManager.Location.DEVICE);
outImg.put(new INDArrayIndex[]{point(exampleNum), all(), all(), all()}, image.getImage());
List<ImageObject> objectsThisImg = objects.get(exampleNum);
label(image, objectsThisImg, outLabel, exampleNum);
} catch (IOException e) {
throw new RuntimeException(e);
}
exampleNum++;
}
if(!nchw) {
outImg = outImg.permute(0, 2, 3, 1); //NCHW to NHWC
outLabel = outLabel.permute(0, 2, 3, 1);
}
return new NDArrayRecordBatch(Arrays.asList(outImg, outLabel));
}
private void label(Image image, List<ImageObject> objectsThisImg, INDArray outLabel, int exampleNum) {
int oW = image.getOrigW();
int oH = image.getOrigH();
int W = oW;
int H = oH;
//put the label data into the output label array
for (ImageObject io : objectsThisImg) {
double cx = io.getXCenterPixels();
double cy = io.getYCenterPixels();
if (imageTransform != null) {
W = imageTransform.getCurrentImage().getWidth();
H = imageTransform.getCurrentImage().getHeight();
float[] pts = imageTransform.query(io.getX1(), io.getY1(), io.getX2(), io.getY2());
int minX = Math.round(Math.min(pts[0], pts[2]));
int maxX = Math.round(Math.max(pts[0], pts[2]));
int minY = Math.round(Math.min(pts[1], pts[3]));
int maxY = Math.round(Math.max(pts[1], pts[3]));
io = new ImageObject(minX, minY, maxX, maxY, io.getLabel());
cx = io.getXCenterPixels();
cy = io.getYCenterPixels();
if (cx < 0 || cx >= W || cy < 0 || cy >= H) {
continue;
}
}
double[] cxyPostScaling = ImageUtils.translateCoordsScaleImage(cx, cy, W, H, width, height);
double[] tlPost = ImageUtils.translateCoordsScaleImage(io.getX1(), io.getY1(), W, H, width, height);
double[] brPost = ImageUtils.translateCoordsScaleImage(io.getX2(), io.getY2(), W, H, width, height);
//Get grid position for image
int imgGridX = (int) (cxyPostScaling[0] / width * gridW);
int imgGridY = (int) (cxyPostScaling[1] / height * gridH);
//Convert pixels to grid position, for TL and BR X/Y
tlPost[0] = tlPost[0] / width * gridW;
tlPost[1] = tlPost[1] / height * gridH;
brPost[0] = brPost[0] / width * gridW;
brPost[1] = brPost[1] / height * gridH;
//Put TL, BR into label array:
Preconditions.checkState(imgGridY >= 0 && imgGridY < outLabel.size(2), "Invalid image center in Y axis: "
+ "calculated grid location of %s, must be between 0 (inclusive) and %s (exclusive). Object label center is outside "
+ "of image bounds. Image object: %s", imgGridY, outLabel.size(2), io);
Preconditions.checkState(imgGridX >= 0 && imgGridX < outLabel.size(3), "Invalid image center in X axis: "
+ "calculated grid location of %s, must be between 0 (inclusive) and %s (exclusive). Object label center is outside "
+ "of image bounds. Image object: %s", imgGridY, outLabel.size(2), io);
outLabel.putScalar(exampleNum, 0, imgGridY, imgGridX, tlPost[0]);
outLabel.putScalar(exampleNum, 1, imgGridY, imgGridX, tlPost[1]);
outLabel.putScalar(exampleNum, 2, imgGridY, imgGridX, brPost[0]);
outLabel.putScalar(exampleNum, 3, imgGridY, imgGridX, brPost[1]);
//Put label class into label array: (one-hot representation)
int labelIdx = labels.indexOf(io.getLabel());
outLabel.putScalar(exampleNum, 4 + labelIdx, imgGridY, imgGridX, 1.0);
}
}
@Override
public List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException {
invokeListeners(uri);
if (imageLoader == null) {
imageLoader = new NativeImageLoader(height, width, channels, imageTransform);
}
Image image = this.imageLoader.asImageMatrix(dataInputStream);
if(!nchw)
image.setImage(image.getImage().permute(0,2,3,1));
Nd4j.getAffinityManager().ensureLocation(image.getImage(), AffinityManager.Location.DEVICE);
List<Writable> ret = RecordConverter.toRecord(image.getImage());
if (appendLabel) {
List<ImageObject> imageObjectsForPath = labelProvider.getImageObjectsForPath(uri.getPath());
int nClasses = labels.size();
INDArray outLabel = Nd4j.create(1, 4 + nClasses, gridH, gridW);
label(image, imageObjectsForPath, outLabel, 0);
if(!nchw)
outLabel = outLabel.permute(0,2,3,1); //NCHW to NHWC
ret.add(new NDArrayWritable(outLabel));
}
return ret;
}
@Override
public Record nextRecord() {
List<Writable> list = next();
URI uri = URIUtil.fileToURI(currentFile);
return new org.datavec.api.records.impl.Record(list, new RecordMetaDataImageURI(uri, BaseImageRecordReader.class,
currentImage.getOrigC(), currentImage.getOrigH(), currentImage.getOrigW()));
}
}
@@ -0,0 +1,169 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.recordreader.objdetect.impl;
import org.bytedeco.javacpp.BytePointer;
import org.bytedeco.javacpp.IntPointer;
import org.bytedeco.javacpp.Loader;
import org.bytedeco.javacpp.Pointer;
import org.bytedeco.javacpp.PointerPointer;
import org.datavec.image.recordreader.objdetect.ImageObject;
import org.datavec.image.recordreader.objdetect.ImageObjectLabelProvider;
import org.bytedeco.hdf5.*;
import static org.bytedeco.hdf5.global.hdf5.*;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class SvhnLabelProvider implements ImageObjectLabelProvider {
private static DataType refType = new DataType(PredType.STD_REF_OBJ());
private static DataType charType = new DataType(PredType.NATIVE_CHAR());
private static DataType intType = new DataType(PredType.NATIVE_INT());
private Map<String, List<ImageObject>> labelMap;
public SvhnLabelProvider(File dir) throws IOException {
labelMap = new HashMap<String, List<ImageObject>>();
H5File file = new H5File(dir.getPath() + "/digitStruct.mat", H5F_ACC_RDONLY());
Group group = file.openGroup("digitStruct");
DataSet nameDataset = group.openDataSet("name");
DataSpace nameSpace = nameDataset.getSpace();
DataSet bboxDataset = group.openDataSet("bbox");
DataSpace bboxSpace = bboxDataset.getSpace();
long[] dims = new long[2];
bboxSpace.getSimpleExtentDims(dims);
int n = (int)(dims[0] * dims[1]);
int ptrSize = Loader.sizeof(Pointer.class);
PointerPointer namePtr = new PointerPointer(n);
PointerPointer bboxPtr = new PointerPointer(n);
nameDataset.read(namePtr, refType);
bboxDataset.read(bboxPtr, refType);
BytePointer bytePtr = new BytePointer(256);
PointerPointer topPtr = new PointerPointer(256);
PointerPointer leftPtr = new PointerPointer(256);
PointerPointer heightPtr = new PointerPointer(256);
PointerPointer widthPtr = new PointerPointer(256);
PointerPointer labelPtr = new PointerPointer(256);
IntPointer intPtr = new IntPointer(256);
for (int i = 0; i < n; i++) {
DataSet nameRef = new DataSet(file, namePtr.position(i * ptrSize));
nameRef.read(bytePtr, charType);
String filename = bytePtr.getString();
Group bboxGroup = new Group(file, bboxPtr.position(i * ptrSize));
DataSet topDataset = bboxGroup.openDataSet("top");
DataSet leftDataset = bboxGroup.openDataSet("left");
DataSet heightDataset = bboxGroup.openDataSet("height");
DataSet widthDataset = bboxGroup.openDataSet("width");
DataSet labelDataset = bboxGroup.openDataSet("label");
DataSpace topSpace = topDataset.getSpace();
topSpace.getSimpleExtentDims(dims);
int m = (int)(dims[0] * dims[1]);
ArrayList<ImageObject> list = new ArrayList<ImageObject>(m);
boolean isFloat = topDataset.asAbstractDs().getTypeClass() == H5T_FLOAT;
if (!isFloat) {
topDataset.read(topPtr.position(0), refType);
leftDataset.read(leftPtr.position(0), refType);
heightDataset.read(heightPtr.position(0), refType);
widthDataset.read(widthPtr.position(0), refType);
labelDataset.read(labelPtr.position(0), refType);
}
assert !isFloat || m == 1;
for (int j = 0; j < m; j++) {
DataSet topSet = isFloat ? topDataset : new DataSet(file, topPtr.position(j * ptrSize));
topSet.read(intPtr, intType);
int top = intPtr.get();
DataSet leftSet = isFloat ? leftDataset : new DataSet(file, leftPtr.position(j * ptrSize));
leftSet.read(intPtr, intType);
int left = intPtr.get();
DataSet heightSet = isFloat ? heightDataset : new DataSet(file, heightPtr.position(j * ptrSize));
heightSet.read(intPtr, intType);
int height = intPtr.get();
DataSet widthSet = isFloat ? widthDataset : new DataSet(file, widthPtr.position(j * ptrSize));
widthSet.read(intPtr, intType);
int width = intPtr.get();
DataSet labelSet = isFloat ? labelDataset : new DataSet(file, labelPtr.position(j * ptrSize));
labelSet.read(intPtr, intType);
int label = intPtr.get();
if (label == 10) {
label = 0;
}
list.add(new ImageObject(left, top, left + width, top + height, Integer.toString(label)));
topSet.deallocate();
leftSet.deallocate();
heightSet.deallocate();
widthSet.deallocate();
labelSet.deallocate();
}
topSpace.deallocate();
if (!isFloat) {
topDataset.deallocate();
leftDataset.deallocate();
heightDataset.deallocate();
widthDataset.deallocate();
labelDataset.deallocate();
}
nameRef.deallocate();
bboxGroup.deallocate();
labelMap.put(filename, list);
}
nameSpace.deallocate();
bboxSpace.deallocate();
nameDataset.deallocate();
bboxDataset.deallocate();
group.deallocate();
file.deallocate();
}
@Override
public List<ImageObject> getImageObjectsForPath(String path) {
File file = new File(path);
String filename = file.getName();
return labelMap.get(filename);
}
@Override
public List<ImageObject> getImageObjectsForPath(URI uri) {
return getImageObjectsForPath(uri.toString());
}
}
@@ -0,0 +1,147 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.recordreader.objdetect.impl;
import lombok.NonNull;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.datavec.image.recordreader.objdetect.ImageObject;
import org.datavec.image.recordreader.objdetect.ImageObjectLabelProvider;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
public class VocLabelProvider implements ImageObjectLabelProvider {
private static final String OBJECT_START_TAG = "<object>";
private static final String OBJECT_END_TAG = "</object>";
private static final String NAME_TAG = "<name>";
private static final String XMIN_TAG = "<xmin>";
private static final String YMIN_TAG = "<ymin>";
private static final String XMAX_TAG = "<xmax>";
private static final String YMAX_TAG = "<ymax>";
private String annotationsDir;
public VocLabelProvider(@NonNull String baseDirectory){
this.annotationsDir = FilenameUtils.concat(baseDirectory, "Annotations");
if(!new File(annotationsDir).exists()){
throw new IllegalStateException("Annotations directory does not exist. Annotation files should be " +
"present at baseDirectory/Annotations/nnnnnn.xml. Expected location: " + annotationsDir);
}
}
@Override
public List<ImageObject> getImageObjectsForPath(String path) {
int idx = path.lastIndexOf('/');
idx = Math.max(idx, path.lastIndexOf('\\'));
String filename = path.substring(idx+1, path.length()-4); //-4: ".jpg"
String xmlPath = FilenameUtils.concat(annotationsDir, filename + ".xml");
File xmlFile = new File(xmlPath);
if(!xmlFile.exists()){
throw new IllegalStateException("Could not find XML file for image " + path + "; expected at " + xmlPath);
}
String xmlContent;
try{
xmlContent = FileUtils.readFileToString(xmlFile);
} catch (IOException e){
throw new RuntimeException(e);
}
//Normally we'd use Jackson to parse XML, but Jackson has real trouble with multiple XML elements with
// the same name. However, the structure is simple and we can parse it manually (even though it's not
// the most elegant thing to do :)
String[] lines = xmlContent.split("\n");
List<ImageObject> out = new ArrayList<>();
for( int i=0; i<lines.length; i++ ){
if(!lines[i].contains(OBJECT_START_TAG)){
continue;
}
String name = null;
int xmin = Integer.MIN_VALUE;
int ymin = Integer.MIN_VALUE;
int xmax = Integer.MIN_VALUE;
int ymax = Integer.MIN_VALUE;
while(!lines[i].contains(OBJECT_END_TAG)){
if(name == null && lines[i].contains(NAME_TAG)){
int idxStartName = lines[i].indexOf('>') + 1;
int idxEndName = lines[i].lastIndexOf('<');
name = lines[i].substring(idxStartName, idxEndName);
i++;
continue;
}
if(xmin == Integer.MIN_VALUE && lines[i].contains(XMIN_TAG)){
xmin = extractAndParse(lines[i]);
i++;
continue;
}
if(ymin == Integer.MIN_VALUE && lines[i].contains(YMIN_TAG)){
ymin = extractAndParse(lines[i]);
i++;
continue;
}
if(xmax == Integer.MIN_VALUE && lines[i].contains(XMAX_TAG)){
xmax = extractAndParse(lines[i]);
i++;
continue;
}
if(ymax == Integer.MIN_VALUE && lines[i].contains(YMAX_TAG)){
ymax = extractAndParse(lines[i]);
i++;
continue;
}
i++;
}
if(name == null){
throw new IllegalStateException("Invalid object format: no name tag found for object in file " + xmlPath);
}
if(xmin == Integer.MIN_VALUE || ymin == Integer.MIN_VALUE || xmax == Integer.MIN_VALUE || ymax == Integer.MIN_VALUE){
throw new IllegalStateException("Invalid object format: did not find all of xmin/ymin/xmax/ymax tags in " + xmlPath);
}
out.add(new ImageObject(xmin, ymin, xmax, ymax, name));
}
return out;
}
private int extractAndParse(String line){
int idxStartName = line.indexOf('>') + 1;
int idxEndName = line.lastIndexOf('<');
String substring = line.substring(idxStartName, idxEndName);
return Integer.parseInt(substring);
}
@Override
public List<ImageObject> getImageObjectsForPath(URI uri) {
return getImageObjectsForPath(uri.toString());
}
}
@@ -0,0 +1,65 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.bytedeco.javacv.FrameConverter;
import org.datavec.image.data.ImageWritable;
import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties;
import java.util.Random;
@NoArgsConstructor
@JsonIgnoreProperties({"converter", "currentImage"})
@Data
public abstract class BaseImageTransform<F> implements ImageTransform {
protected Random random;
protected FrameConverter<F> converter;
protected ImageWritable currentImage;
protected BaseImageTransform(Random random) {
this.random = random;
}
@Override
public final ImageWritable transform(ImageWritable image) {
return transform(image, random);
}
@Override
public final ImageWritable transform(ImageWritable image, Random random) {
return currentImage = doTransform(image, random);
}
protected abstract ImageWritable doTransform(ImageWritable image, Random random);
@Override
public float[] query(float... coordinates) {
throw new UnsupportedOperationException();
}
@Override
public ImageWritable getCurrentImage() {
return currentImage;
}
}
@@ -0,0 +1,123 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.datavec.image.data.ImageWritable;
import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties;
import org.nd4j.shade.jackson.annotation.JsonInclude;
import org.nd4j.shade.jackson.annotation.JsonProperty;
import java.util.Random;
import org.bytedeco.opencv.opencv_core.*;
@Accessors(fluent = true)
@JsonIgnoreProperties({"borderValue"})
@JsonInclude(JsonInclude.Include.NON_NULL)
@Data
public class BoxImageTransform extends BaseImageTransform<Mat> {
private int width;
private int height;
private int x;
private int y;
@Getter
@Setter
Scalar borderValue = Scalar.ZERO;
/** Calls {@code this(null, width, height)}. */
public BoxImageTransform(@JsonProperty("width") int width, @JsonProperty("height") int height) {
this(null, width, height);
}
/**
* Constructs an instance of the ImageTransform.
*
* @param random object to use (or null for deterministic)
* @param width of the boxed image (pixels)
* @param height of the boxed image (pixels)
*/
public BoxImageTransform(Random random, int width, int height) {
super(random);
this.width = width;
this.height = height;
this.converter = new OpenCVFrameConverter.ToMat();
}
/**
* Takes an image and returns a boxed version of the image.
*
* @param image to transform, null == end of stream
* @param random object to use (or null for deterministic)
* @return transformed image
*/
@Override
protected ImageWritable doTransform(ImageWritable image, Random random) {
if (image == null) {
return null;
}
Mat mat = converter.convert(image.getFrame());
Mat box = new Mat(height, width, mat.type());
box.put(borderValue);
x = (mat.cols() - width) / 2;
y = (mat.rows() - height) / 2;
int w = Math.min(mat.cols(), width);
int h = Math.min(mat.rows(), height);
Rect matRect = new Rect(x, y, w, h);
Rect boxRect = new Rect(x, y, w, h);
if (x <= 0) {
matRect.x(0);
boxRect.x(-x);
} else {
matRect.x(x);
boxRect.x(0);
}
if (y <= 0) {
matRect.y(0);
boxRect.y(-y);
} else {
matRect.y(y);
boxRect.y(0);
}
mat.apply(matRect).copyTo(box.apply(boxRect));
return new ImageWritable(converter.convert(box));
}
@Override
public float[] query(float... coordinates) {
float[] transformed = new float[coordinates.length];
for (int i = 0; i < coordinates.length; i += 2) {
transformed[i ] = coordinates[i ] - x;
transformed[i + 1] = coordinates[i + 1] - y;
}
return transformed;
}
}
@@ -0,0 +1,102 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.datavec.image.data.ImageWritable;
import org.nd4j.shade.jackson.annotation.JsonInclude;
import java.util.Random;
import org.bytedeco.opencv.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_imgproc.*;
@JsonInclude(JsonInclude.Include.NON_NULL)
@Data
public class ColorConversionTransform extends BaseImageTransform {
/**
* Color Conversion code
* {@link org.bytedeco.opencv.global.opencv_imgproc}
*/
private int conversionCode;
/**
* Default conversion BGR to Luv (chroma) color.
*/
public ColorConversionTransform() {
this(new Random(1234), COLOR_BGR2Luv);
}
/**
* Return new ColorConversion object
*
* @param conversionCode to transform,
*/
public ColorConversionTransform(int conversionCode) {
this(null, conversionCode);
}
/**
* Return new ColorConversion object
*
* @param random Random
* @param conversionCode to transform,
*/
public ColorConversionTransform(Random random, int conversionCode) {
super(random);
this.conversionCode = conversionCode;
converter = new OpenCVFrameConverter.ToMat();
}
/**
* Takes an image and returns a transformed image.
* Uses the random object in the case of random transformations.
*
* @param image to transform, null == end of stream
* @param random object to use (or null for deterministic)
* @return transformed image
*/
@Override
protected ImageWritable doTransform(ImageWritable image, Random random) {
if (image == null) {
return null;
}
Mat mat = (Mat) converter.convert(image.getFrame());
Mat result = new Mat();
try {
cvtColor(mat, result, conversionCode);
} catch (Exception e) {
throw new RuntimeException(e);
}
return new ImageWritable(converter.convert(result));
}
@Override
public float[] query(float... coordinates) {
return coordinates;
}
}
@@ -0,0 +1,116 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.datavec.image.data.ImageWritable;
import org.nd4j.shade.jackson.annotation.JsonInclude;
import org.nd4j.shade.jackson.annotation.JsonProperty;
import java.util.Random;
import org.bytedeco.opencv.opencv_core.*;
@JsonInclude(JsonInclude.Include.NON_NULL)
@Data
public class CropImageTransform extends BaseImageTransform<Mat> {
private int cropTop;
private int cropLeft;
private int cropBottom;
private int cropRight;
private int x;
private int y;
/** Calls {@code this(null, crop, crop, crop, crop)}. */
public CropImageTransform(int crop) {
this(null, crop, crop, crop, crop);
}
/** Calls {@code this(random, crop, crop, crop, crop)}. */
public CropImageTransform(Random random, int crop) {
this(random, crop, crop, crop, crop);
}
/** Calls {@code this(random, cropTop, cropLeft, cropBottom, cropRight)}. */
public CropImageTransform(@JsonProperty("cropTop") int cropTop, @JsonProperty("cropLeft") int cropLeft,
@JsonProperty("cropBottom") int cropBottom, @JsonProperty("cropRight") int cropRight) {
this(null, cropTop, cropLeft, cropBottom, cropRight);
}
/**
* Constructs an instance of the ImageTransform.
*
* @param random object to use (or null for deterministic)
* @param cropTop maximum cropping of the top of the image (pixels)
* @param cropLeft maximum cropping of the left of the image (pixels)
* @param cropBottom maximum cropping of the bottom of the image (pixels)
* @param cropRight maximum cropping of the right of the image (pixels)
*/
public CropImageTransform(Random random, int cropTop, int cropLeft, int cropBottom, int cropRight) {
super(random);
this.cropTop = cropTop;
this.cropLeft = cropLeft;
this.cropBottom = cropBottom;
this.cropRight = cropRight;
this.converter = new OpenCVFrameConverter.ToMat();
}
/**
* Takes an image and returns a transformed image.
* Uses the random object in the case of random transformations.
*
* @param image to transform, null == end of stream
* @param random object to use (or null for deterministic)
* @return transformed image
*/
@Override
protected ImageWritable doTransform(ImageWritable image, Random random) {
if (image == null) {
return null;
}
Mat mat = converter.convert(image.getFrame());
int top = random != null ? random.nextInt(cropTop + 1) : cropTop;
int left = random != null ? random.nextInt(cropLeft + 1) : cropLeft;
int bottom = random != null ? random.nextInt(cropBottom + 1) : cropBottom;
int right = random != null ? random.nextInt(cropRight + 1) : cropRight;
y = Math.min(top, mat.rows() - 1);
x = Math.min(left, mat.cols() - 1);
int h = Math.max(1, mat.rows() - bottom - y);
int w = Math.max(1, mat.cols() - right - x);
Mat result = mat.apply(new Rect(x, y, w, h));
return new ImageWritable(converter.convert(result));
}
@Override
public float[] query(float... coordinates) {
float[] transformed = new float[coordinates.length];
for (int i = 0; i < coordinates.length; i += 2) {
transformed[i ] = coordinates[i ] - x;
transformed[i + 1] = coordinates[i + 1] - y;
}
return transformed;
}
}
@@ -0,0 +1,112 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.datavec.image.data.ImageWritable;
import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties;
import org.nd4j.shade.jackson.annotation.JsonInclude;
import java.util.Random;
import org.bytedeco.opencv.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_imgproc.*;
@JsonIgnoreProperties({"splitChannels", "converter"})
@JsonInclude(JsonInclude.Include.NON_NULL)
@Data
public class EqualizeHistTransform extends BaseImageTransform {
/**
* Color Conversion code
* {@link org.bytedeco.opencv.global.opencv_imgproc}
*/
private int conversionCode;
private MatVector splitChannels = new MatVector();
/**
* Default transforms histogram equalization for CV_BGR2GRAY (grayscale)
*/
public EqualizeHistTransform() {
this(new Random(1234), CV_BGR2GRAY);
}
/**
* Return contrast normalized object
*
* @param conversionCode to transform,
*/
public EqualizeHistTransform(int conversionCode) {
this(null, conversionCode);
}
/**
* Return contrast normalized object
*
* @param random Random
* @param conversionCode to transform,
*/
public EqualizeHistTransform(Random random, int conversionCode) {
super(random);
this.conversionCode = conversionCode;
this.converter = new OpenCVFrameConverter.ToMat();
}
/**
* Takes an image and returns a transformed image.
* Uses the random object in the case of random transformations.
*
* @param image to transform, null == end of stream
* @param random object to use (or null for deterministic)
* @return transformed image
*/
@Override
protected ImageWritable doTransform(ImageWritable image, Random random) {
if (image == null) {
return null;
}
Mat mat = (Mat) converter.convert(image.getFrame());
Mat result = new Mat();
try {
if (mat.channels() == 1) {
equalizeHist(mat, result);
} else {
split(mat, splitChannels);
equalizeHist(splitChannels.get(0), splitChannels.get(0)); //equalize histogram on the 1st channel (Y)
merge(splitChannels, result);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
return new ImageWritable(converter.convert(result));
}
@Override
public float[] query(float... coordinates) {
return coordinates;
}
}
@@ -0,0 +1,133 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.datavec.image.data.ImageWritable;
import org.nd4j.shade.jackson.annotation.JsonInclude;
import java.util.Random;
import org.bytedeco.opencv.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_core.*;
@JsonInclude(JsonInclude.Include.NON_NULL)
@Data
public class FlipImageTransform extends BaseImageTransform<Mat> {
/**
* the deterministic flip mode
* {@code 0} Flips around x-axis.
* {@code >0} Flips around y-axis.
* {@code <0} Flips around both axes.
*/
private int flipMode;
private int h;
private int w;
private int mode;
/**
* Calls {@code this(null)}.
*/
public FlipImageTransform() {
this(null);
}
/**
* Calls {@code this(null)} and sets the flip mode.
*
* @param flipMode the deterministic flip mode
* {@code 0} Flips around x-axis.
* {@code >0} Flips around y-axis.
* {@code <0} Flips around both axes.
*/
public FlipImageTransform(int flipMode) {
this(null);
this.flipMode = flipMode;
}
/**
* Constructs an instance of the ImageTransform. Randomly does not flip,
* or flips horizontally or vertically, or both.
*
* @param random object to use (or null for deterministic)
*/
public FlipImageTransform(Random random) {
super(random);
converter = new OpenCVFrameConverter.ToMat();
}
@Override
protected ImageWritable doTransform(ImageWritable image, Random random) {
if (image == null) {
return null;
}
Mat mat = converter.convert(image.getFrame());
if(mat == null) {
return null;
}
h = mat.rows();
w = mat.cols();
mode = random != null ? random.nextInt(4) - 2 : flipMode;
Mat result = new Mat();
if (mode < -1) {
// no flip
mat.copyTo(result);
} else {
flip(mat, result, mode);
}
return new ImageWritable(converter.convert(result));
}
@Override
public float[] query(float... coordinates) {
float[] transformed = new float[coordinates.length];
for (int i = 0; i < coordinates.length; i += 2) {
float x = coordinates[i ];
float y = coordinates[i + 1];
float x2 = w - x - 1;
float y2 = h - y - 1;
if (mode < -1) {
transformed[i ] = x;
transformed[i + 1] = y;
} else if (mode == 0) {
transformed[i ] = x;
transformed[i + 1] = y2;
} else if (mode > 0) {
transformed[i ] = x2;
transformed[i + 1] = y;
} else if (mode < 0) {
transformed[i ] = x2;
transformed[i + 1] = y2;
}
}
return transformed;
}
}
@@ -0,0 +1,58 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import org.datavec.api.transform.Operation;
import org.datavec.image.data.ImageWritable;
import org.nd4j.shade.jackson.annotation.JsonInclude;
import org.nd4j.shade.jackson.annotation.JsonTypeInfo;
import java.util.Random;
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class")
public interface ImageTransform extends Operation<ImageWritable, ImageWritable> {
/**
* Takes an image and returns a transformed image.
* Uses the random object in the case of random transformations.
*
* @param image to transform, null == end of stream
* @param random object to use (or null for deterministic)
* @return transformed image
*/
ImageWritable transform(ImageWritable image, Random random);
/**
* Transforms the given coordinates using the parameters that were used to transform the last image.
*
* @param coordinates to transforms (x1, y1, x2, y2, ...)
* @return transformed coordinates
*/
float[] query(float... coordinates);
/**
* Returns the last transformed image or null if none transformed yet.
*
* @return Last transformed image or null
*/
ImageWritable getCurrentImage();
}
@@ -0,0 +1,256 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.datavec.api.transform.serde.JsonMappers;
import org.datavec.api.writable.Writable;
import org.datavec.image.data.ImageWritable;
import org.datavec.image.loader.NativeImageLoader;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.shade.jackson.core.JsonProcessingException;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
@Data
@Slf4j
@NoArgsConstructor
public class ImageTransformProcess {
private List<ImageTransform> transformList;
private int seed;
public ImageTransformProcess(int seed, ImageTransform... transforms) {
this.seed = seed;
this.transformList = Arrays.asList(transforms);
}
public ImageTransformProcess(int seed, List<ImageTransform> transformList) {
this.seed = seed;
this.transformList = transformList;
}
public ImageTransformProcess(Builder builder) {
this(builder.seed, builder.transformList);
}
public List<Writable> execute(List<Writable> image) {
throw new UnsupportedOperationException();
}
public INDArray executeArray(ImageWritable image) throws IOException {
Random random = null;
if (seed != 0) {
random = new Random(seed);
}
ImageWritable currentImage = image;
for (ImageTransform transform : transformList) {
currentImage = transform.transform(currentImage, random);
}
NativeImageLoader imageLoader = new NativeImageLoader();
return imageLoader.asMatrix(currentImage);
}
public ImageWritable execute(ImageWritable image) throws IOException {
Random random = null;
if (seed != 0) {
random = new Random(seed);
}
ImageWritable currentImage = image;
for (ImageTransform transform : transformList) {
currentImage = transform.transform(currentImage, random);
}
return currentImage;
}
public ImageWritable transformFileUriToInput(URI uri) throws IOException {
NativeImageLoader imageLoader = new NativeImageLoader();
ImageWritable img = imageLoader.asWritable(new File(uri));
return img;
}
/**
* Convert the ImageTransformProcess to a JSON string
*
* @return ImageTransformProcess, as JSON
*/
public String toJson() {
try {
return JsonMappers.getMapper().writeValueAsString(this);
} catch (JsonProcessingException e) {
//TODO better exceptions
throw new RuntimeException(e);
}
}
/**
* Convert the ImageTransformProcess to a YAML string
*
* @return ImageTransformProcess, as YAML
*/
public String toYaml() {
try {
return JsonMappers.getMapperYaml().writeValueAsString(this);
} catch (JsonProcessingException e) {
//TODO better exceptions
throw new RuntimeException(e);
}
}
/**
* Deserialize a JSON String (created by {@link #toJson()}) to a ImageTransformProcess
*
* @return ImageTransformProcess, from JSON
*/
public static ImageTransformProcess fromJson(String json) {
try {
return JsonMappers.getMapper().readValue(json, ImageTransformProcess.class);
} catch (IOException e) {
//TODO better exceptions
throw new RuntimeException(e);
}
}
/**
* Deserialize a JSON String (created by {@link #toJson()}) to a ImageTransformProcess
*
* @return ImageTransformProcess, from JSON
*/
public static ImageTransformProcess fromYaml(String yaml) {
try {
return JsonMappers.getMapperYaml().readValue(yaml, ImageTransformProcess.class);
} catch (IOException e) {
//TODO better exceptions
throw new RuntimeException(e);
}
}
/**
* Builder class for constructing a ImageTransformProcess
*/
public static class Builder {
private List<ImageTransform> transformList;
private int seed = 0;
public Builder() {
transformList = new ArrayList<>();
}
public Builder seed(int seed) {
this.seed = seed;
return this;
}
public Builder cropImageTransform(int crop) {
transformList.add(new CropImageTransform(crop));
return this;
}
public Builder cropImageTransform(int cropTop, int cropLeft, int cropBottom, int cropRight) {
transformList.add(new CropImageTransform(cropTop, cropLeft, cropBottom, cropRight));
return this;
}
public Builder colorConversionTransform(int conversionCode) {
transformList.add(new ColorConversionTransform(conversionCode));
return this;
}
public Builder equalizeHistTransform(int conversionCode) {
transformList.add(new EqualizeHistTransform(conversionCode));
return this;
}
public Builder flipImageTransform(int flipMode) {
transformList.add(new FlipImageTransform(flipMode));
return this;
}
public Builder randomCropTransform(int height, int width) {
transformList.add(new RandomCropTransform(height, width));
return this;
}
public Builder randomCropTransform(long seed, int height, int width) {
transformList.add(new RandomCropTransform(seed, height, width));
return this;
}
public Builder resizeImageTransform(int newWidth, int newHeight) {
transformList.add(new ResizeImageTransform(newWidth, newHeight));
return this;
}
public Builder rotateImageTransform(float angle) {
transformList.add(new RotateImageTransform(angle));
return this;
}
public Builder rotateImageTransform(float centerx, float centery, float angle, float scale) {
transformList.add(new RotateImageTransform(centerx, centery, angle, scale));
return this;
}
public Builder scaleImageTransform(float delta) {
transformList.add(new ScaleImageTransform(delta));
return this;
}
public Builder scaleImageTransform(float dx, float dy) {
transformList.add(new ScaleImageTransform(dx, dy));
return this;
}
public Builder warpImageTransform(float delta) {
transformList.add(new WarpImageTransform(delta));
return this;
}
public Builder warpImageTransform(float dx1, float dy1, float dx2, float dy2, float dx3, float dy3, float dx4,
float dy4) {
transformList.add(new WarpImageTransform(dx1, dy1, dx2, dy2, dx3, dy3, dx4, dy4));
return this;
}
public ImageTransformProcess build() {
return new ImageTransformProcess(this);
}
}
}
@@ -0,0 +1,141 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.datavec.image.data.ImageWritable;
import org.nd4j.linalg.factory.Nd4j;
import java.util.Random;
import org.bytedeco.opencv.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_imgproc.*;
@Data
public class LargestBlobCropTransform extends BaseImageTransform<Mat> {
protected org.nd4j.linalg.api.rng.Random rng;
protected int mode, method, blurWidth, blurHeight, upperThresh, lowerThresh;
protected boolean isCanny;
private int x;
private int y;
/** Calls {@code this(null}*/
public LargestBlobCropTransform() {
this(null);
}
/** Calls {@code this(random, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE, 3, 3, 100, 200, true)}*/
public LargestBlobCropTransform(Random random) {
this(random, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE, 3, 3, 100, 200, true);
}
/**
*
* @param random Object to use (or null for deterministic)
* @param mode Contour retrieval mode
* @param method Contour approximation method
* @param blurWidth Width of blurring kernel size
* @param blurHeight Height of blurring kernel size
* @param lowerThresh Lower threshold for either Canny or Threshold
* @param upperThresh Upper threshold for either Canny or Threshold
* @param isCanny Whether the edge detector is Canny or Threshold
*/
public LargestBlobCropTransform(Random random, int mode, int method, int blurWidth, int blurHeight, int lowerThresh,
int upperThresh, boolean isCanny) {
super(random);
this.rng = Nd4j.getRandom();
this.mode = mode;
this.method = method;
this.blurWidth = blurWidth;
this.blurHeight = blurHeight;
this.lowerThresh = lowerThresh;
this.upperThresh = upperThresh;
this.isCanny = isCanny;
this.converter = new OpenCVFrameConverter.ToMat();
}
/**
* Takes an image and returns a cropped image based on it's largest blob.
*
* @param image to transform, null == end of stream
* @param random object to use (or null for deterministic)
* @return transformed image
*/
@Override
protected ImageWritable doTransform(ImageWritable image, Random random) {
if (image == null) {
return null;
}
//Convert image to gray and blur
Mat original = converter.convert(image.getFrame());
Mat grayed = new Mat();
cvtColor(original, grayed, CV_BGR2GRAY);
if (blurWidth > 0 && blurHeight > 0)
blur(grayed, grayed, new Size(blurWidth, blurHeight));
//Get edges from Canny edge detector
Mat edgeOut = new Mat();
if (isCanny)
Canny(grayed, edgeOut, lowerThresh, upperThresh);
else
threshold(grayed, edgeOut, lowerThresh, upperThresh, 0);
double largestArea = 0;
Rect boundingRect = new Rect();
MatVector contours = new MatVector();
Mat hierarchy = new Mat();
findContours(edgeOut, contours, hierarchy, this.mode, this.method);
for (int i = 0; i < contours.size(); i++) {
// Find the area of contour
double area = contourArea(contours.get(i), false);
if (area > largestArea) {
// Find the bounding rectangle for biggest contour
boundingRect = boundingRect(contours.get(i));
}
}
//Apply crop and return result
x = boundingRect.x();
y = boundingRect.y();
Mat result = original.apply(boundingRect);
return new ImageWritable(converter.convert(result));
}
@Override
public float[] query(float... coordinates) {
float[] transformed = new float[coordinates.length];
for (int i = 0; i < coordinates.length; i += 2) {
transformed[i ] = coordinates[i ] - x;
transformed[i + 1] = coordinates[i + 1] - y;
}
return transformed;
}
}
@@ -0,0 +1,52 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import org.datavec.image.data.ImageWritable;
import java.util.Random;
import org.bytedeco.opencv.opencv_core.*;
@Data
public class MultiImageTransform extends BaseImageTransform<Mat> {
private PipelineImageTransform transform;
public MultiImageTransform(ImageTransform... transforms) {
this(null, transforms);
}
public MultiImageTransform(Random random, ImageTransform... transforms) {
super(random);
transform = new PipelineImageTransform(transforms);
}
@Override
protected ImageWritable doTransform(ImageWritable image, Random random) {
return random == null ? transform.transform(image) : transform.transform(image, random);
}
@Override
public float[] query(float... coordinates) {
return transform.query(coordinates);
}
}
@@ -0,0 +1,182 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import lombok.NonNull;
import org.datavec.image.data.ImageWritable;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.common.primitives.Pair;
import java.util.*;
import org.bytedeco.opencv.opencv_core.*;
@Data
public class PipelineImageTransform extends BaseImageTransform<Mat> {
protected List<Pair<ImageTransform, Double>> imageTransforms;
protected boolean shuffle;
protected org.nd4j.linalg.api.rng.Random rng;
protected List<ImageTransform> currentTransforms = new ArrayList<>();
public PipelineImageTransform(ImageTransform... transforms) {
this(1234, false, transforms);
}
public PipelineImageTransform(long seed, boolean shuffle, ImageTransform... transforms) {
super(null); // for perf reasons we ignore java Random, create our own
List<Pair<ImageTransform, Double>> pipeline = new LinkedList<>();
for (int i = 0; i < transforms.length; i++) {
pipeline.add(new Pair<>(transforms[i], 1.0));
}
this.imageTransforms = pipeline;
this.shuffle = shuffle;
this.rng = Nd4j.getRandom();
rng.setSeed(seed);
}
public PipelineImageTransform(List<Pair<ImageTransform, Double>> transforms) {
this(1234, transforms, false);
}
public PipelineImageTransform(List<Pair<ImageTransform, Double>> transforms, boolean shuffle) {
this(1234, transforms, shuffle);
}
public PipelineImageTransform(long seed, List<Pair<ImageTransform, Double>> transforms) {
this(seed, transforms, false);
}
public PipelineImageTransform(long seed, List<Pair<ImageTransform, Double>> transforms, boolean shuffle) {
this(null, seed, transforms, shuffle);
}
public PipelineImageTransform(Random random, long seed, List<Pair<ImageTransform, Double>> transforms,
boolean shuffle) {
super(random); // used by the transforms in the pipeline
this.imageTransforms = transforms;
this.shuffle = shuffle;
this.rng = Nd4j.getRandom();
rng.setSeed(seed);
}
/**
* Takes an image and executes a pipeline of combined transforms.
*
* @param image to transform, null == end of stream
* @param random object to use (or null for deterministic)
* @return transformed image
*/
@Override
protected ImageWritable doTransform(ImageWritable image, Random random) {
if (shuffle) {
Collections.shuffle(imageTransforms);
}
currentTransforms.clear();
// execute each item in the pipeline
for (Pair<ImageTransform, Double> tuple : imageTransforms) {
if (tuple.getSecond() == 1.0 || rng.nextDouble() < tuple.getSecond()) { // probability of execution
currentTransforms.add(tuple.getFirst());
image = random != null ? tuple.getFirst().transform(image, random)
: tuple.getFirst().transform(image);
}
}
return image;
}
@Override
public float[] query(float... coordinates) {
for (ImageTransform transform : currentTransforms) {
coordinates = transform.query(coordinates);
}
return coordinates;
}
/**
* Optional builder helper for PipelineImageTransform
*/
public static class Builder {
protected List<Pair<ImageTransform, Double>> imageTransforms = new ArrayList<>();
protected Long seed = null;
/**
* This method sets RNG seet for this pipeline
*
* @param seed
* @return
*/
public Builder setSeed(long seed) {
this.seed = seed;
return this;
}
/**
* This method adds given transform with 100% invocation probability to this pipelien
*
* @param transform
* @return
*/
public Builder addImageTransform(@NonNull ImageTransform transform) {
return addImageTransform(transform, 1.0);
}
/**
* This method adds given transform with given invocation probability to this pipelien
*
* @param transform
* @param probability
* @return
*/
public Builder addImageTransform(@NonNull ImageTransform transform, Double probability) {
if (probability < 0.0) {
probability = 0.0;
}
if (probability > 1.0) {
probability = 1.0;
}
imageTransforms.add(Pair.makePair(transform, probability));
return this;
}
/**
* This method returns new PipelineImageTransform instance
*
* @return
*/
public PipelineImageTransform build() {
if (seed != null) {
return new PipelineImageTransform(seed, imageTransforms);
} else {
return new PipelineImageTransform(imageTransforms);
}
}
}
}
@@ -0,0 +1,108 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.datavec.image.data.ImageWritable;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties;
import org.nd4j.shade.jackson.annotation.JsonInclude;
import org.nd4j.shade.jackson.annotation.JsonProperty;
import java.util.Random;
import org.bytedeco.opencv.opencv_core.*;
@JsonIgnoreProperties({"rng", "converter"})
@JsonInclude(JsonInclude.Include.NON_NULL)
@Data
public class RandomCropTransform extends BaseImageTransform<Mat> {
protected int outputHeight;
protected int outputWidth;
protected org.nd4j.linalg.api.rng.Random rng;
private int x;
private int y;
public RandomCropTransform(@JsonProperty("outputHeight") int height, @JsonProperty("outputWidth") int width) {
this(1234, height, width);
}
public RandomCropTransform(long seed, int height, int width) {
this(null, seed, height, width);
}
public RandomCropTransform(Random random, long seed, int height, int width) {
super(random);
this.outputHeight = height;
this.outputWidth = width;
this.rng = Nd4j.getRandom();
this.rng.setSeed(seed);
this.converter = new OpenCVFrameConverter.ToMat();
}
/**
* Takes an image and returns a randomly cropped image.
*
* @param image to transform, null == end of stream
* @param random object to use (or null for deterministic)
* @return transformed image
*/
@Override
protected ImageWritable doTransform(ImageWritable image, Random random) {
if (image == null) {
return null;
}
// ensure that transform is valid
if (image.getFrame().imageHeight < outputHeight || image.getFrame().imageWidth < outputWidth)
throw new UnsupportedOperationException(
"Output height/width cannot be more than the input image. Requested: " + outputHeight + "+x"
+ outputWidth + ", got " + image.getFrame().imageHeight + "+x"
+ image.getFrame().imageWidth);
// determine boundary to place random offset
int cropTop = image.getFrame().imageHeight - outputHeight;
int cropLeft = image.getFrame().imageWidth - outputWidth;
Mat mat = converter.convert(image.getFrame());
int top = rng.nextInt(cropTop + 1);
int left = rng.nextInt(cropLeft + 1);
y = Math.min(top, mat.rows() - 1);
x = Math.min(left, mat.cols() - 1);
Mat result = mat.apply(new Rect(x, y, outputWidth, outputHeight));
return new ImageWritable(converter.convert(result));
}
@Override
public float[] query(float... coordinates) {
float[] transformed = new float[coordinates.length];
for (int i = 0; i < coordinates.length; i += 2) {
transformed[i ] = coordinates[i ] - x;
transformed[i + 1] = coordinates[i + 1] - y;
}
return transformed;
}
}
@@ -0,0 +1,100 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.datavec.image.data.ImageWritable;
import org.nd4j.shade.jackson.annotation.JsonInclude;
import org.nd4j.shade.jackson.annotation.JsonProperty;
import java.util.Random;
import org.bytedeco.opencv.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_imgproc.*;
@JsonInclude(JsonInclude.Include.NON_NULL)
@Data
public class ResizeImageTransform extends BaseImageTransform<Mat> {
private int newHeight;
private int newWidth;
private int srch;
private int srcw;
/**
* Returns new ResizeImageTransform object
*
* @param newWidth new Width for the outcome images
* @param newHeight new Height for outcome images
*/
public ResizeImageTransform(@JsonProperty("newWidth") int newWidth, @JsonProperty("newHeight") int newHeight) {
this(null, newWidth, newHeight);
}
/**
* Returns new ResizeImageTransform object
*
* @param random Random
* @param newWidth new Width for the outcome images
* @param newHeight new Height for outcome images
*/
public ResizeImageTransform(Random random, int newWidth, int newHeight) {
super(random);
this.newWidth = newWidth;
this.newHeight = newHeight;
this.converter = new OpenCVFrameConverter.ToMat();
}
/**
* Takes an image and returns a transformed image.
* Uses the random object in the case of random transformations.
*
* @param image to transform, null == end of stream
* @param random object to use (or null for deterministic)
* @return transformed image
*/
@Override
protected ImageWritable doTransform(ImageWritable image, Random random) {
if (image == null) {
return null;
}
Mat mat = converter.convert(image.getFrame());
Mat result = new Mat();
srch = mat.rows();
srcw = mat.cols();
resize(mat, result, new Size(newWidth, newHeight));
return new ImageWritable(converter.convert(result));
}
@Override
public float[] query(float... coordinates) {
float[] transformed = new float[coordinates.length];
for (int i = 0; i < coordinates.length; i += 2) {
transformed[i ] = newWidth * coordinates[i ] / srcw;
transformed[i + 1] = newHeight * coordinates[i + 1] / srch;
}
return transformed;
}
}
@@ -0,0 +1,133 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
import org.bytedeco.javacpp.FloatPointer;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.datavec.image.data.ImageWritable;
import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties;
import org.nd4j.shade.jackson.annotation.JsonInclude;
import org.nd4j.shade.jackson.annotation.JsonProperty;
import java.nio.FloatBuffer;
import java.util.Random;
import org.bytedeco.opencv.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_imgproc.*;
@Accessors(fluent = true)
@JsonIgnoreProperties({"interMode", "borderMode", "borderValue", "converter"})
@JsonInclude(JsonInclude.Include.NON_NULL)
@Data
public class RotateImageTransform extends BaseImageTransform<Mat> {
private float centerx;
private float centery;
private float angle;
private float scale;
@Getter
@Setter
private int interMode = INTER_LINEAR;
@Getter
@Setter
private int borderMode = BORDER_CONSTANT;
@Getter
@Setter
private Scalar borderValue = Scalar.ZERO;
private Mat M;
/** Calls {@code this(null, 0, 0, angle, 0)}. */
public RotateImageTransform(float angle) {
this(null, 0, 0, angle, 0);
}
/** Calls {@code this(random, 0, 0, angle, 0)}. */
public RotateImageTransform(Random random, float angle) {
this(random, 0, 0, angle, 0);
}
/**
* Constructs an instance of the ImageTransform.
*
* @param centerx maximum deviation in x of center of rotation (relative to image center)
* @param centery maximum deviation in y of center of rotation (relative to image center)
* @param angle maximum rotation (degrees)
* @param scale maximum scaling (relative to 1)
*/
public RotateImageTransform(@JsonProperty("centerx") float centerx, @JsonProperty("centery") float centery,
@JsonProperty("angle") float angle, @JsonProperty("scale") float scale) {
this(null, centerx, centery, angle, scale);
}
/**
* Constructs an instance of the ImageTransform.
*
* @param random object to use (or null for deterministic)
* @param centerx maximum deviation in x of center of rotation (relative to image center)
* @param centery maximum deviation in y of center of rotation (relative to image center)
* @param angle maximum rotation (degrees)
* @param scale maximum scaling (relative to 1)
*/
public RotateImageTransform(Random random, float centerx, float centery, float angle, float scale) {
super(random);
this.centerx = centerx;
this.centery = centery;
this.angle = angle;
this.scale = scale;
this.converter = new OpenCVFrameConverter.ToMat();
}
@Override
protected ImageWritable doTransform(ImageWritable image, Random random) {
if (image == null) {
return null;
}
Mat mat = converter.convert(image.getFrame());
float cy = mat.rows() / 2 + centery * (random != null ? 2 * random.nextFloat() - 1 : 1);
float cx = mat.cols() / 2 + centerx * (random != null ? 2 * random.nextFloat() - 1 : 1);
float a = angle * (random != null ? 2 * random.nextFloat() - 1 : 1);
float s = 1 + scale * (random != null ? 2 * random.nextFloat() - 1 : 1);
Mat result = new Mat();
M = getRotationMatrix2D(new Point2f(cx, cy), a, s);
warpAffine(mat, result, M, mat.size(), interMode, borderMode, borderValue);
return new ImageWritable(converter.convert(result));
}
@Override
public float[] query(float... coordinates) {
Mat src = new Mat(1, coordinates.length / 2, CV_32FC2, new FloatPointer(coordinates));
Mat dst = new Mat();
org.bytedeco.opencv.global.opencv_core.transform(src, dst, M);
FloatBuffer buf = dst.createBuffer();
float[] transformed = new float[coordinates.length];
buf.get(transformed);
return transformed;
}
}
@@ -0,0 +1,99 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.datavec.image.data.ImageWritable;
import org.nd4j.shade.jackson.annotation.JsonInclude;
import org.nd4j.shade.jackson.annotation.JsonProperty;
import java.util.Random;
import org.bytedeco.opencv.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_imgproc.*;
@JsonInclude(JsonInclude.Include.NON_NULL)
@Data
public class ScaleImageTransform extends BaseImageTransform<Mat> {
private float dx;
private float dy;
private int srch, h;
private int srcw, w;
/** Calls {@code this(null, delta, delta)}. */
public ScaleImageTransform(float delta) {
this(null, delta, delta);
}
/** Calls {@code this(random, delta, delta)}. */
public ScaleImageTransform(Random random, float delta) {
this(random, delta, delta);
}
/** Calls {@code this(null, dx, dy)}. */
public ScaleImageTransform(@JsonProperty("dx") float dx, @JsonProperty("dy") float dy) {
this(null, dx, dy);
}
/**
* Constructs an instance of the ImageTransform.
*
* @param random object to use (or null for deterministic)
* @param dx maximum scaling in width of the image (pixels)
* @param dy maximum scaling in height of the image (pixels)
*/
public ScaleImageTransform(Random random, float dx, float dy) {
super(random);
this.dx = dx;
this.dy = dy;
this.converter = new OpenCVFrameConverter.ToMat();
}
@Override
protected ImageWritable doTransform(ImageWritable image, Random random) {
if (image == null) {
return null;
}
Mat mat = converter.convert(image.getFrame());
srch = mat.rows();
srcw = mat.cols();
h = Math.round(mat.rows() + dy * (random != null ? 2 * random.nextFloat() - 1 : 1));
w = Math.round(mat.cols() + dx * (random != null ? 2 * random.nextFloat() - 1 : 1));
Mat result = new Mat();
resize(mat, result, new Size(w, h));
return new ImageWritable(converter.convert(result));
}
@Override
public float[] query(float... coordinates) {
float[] transformed = new float[coordinates.length];
for (int i = 0; i < coordinates.length; i += 2) {
transformed[i ] = w * coordinates[i ] / srcw;
transformed[i + 1] = h * coordinates[i + 1] / srch;
}
return transformed;
}
}
@@ -0,0 +1,104 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import org.bytedeco.javacv.CanvasFrame;
import org.bytedeco.javacv.Frame;
import org.datavec.image.data.ImageWritable;
import javax.swing.*;
import java.util.Random;
@Data
public class ShowImageTransform extends BaseImageTransform {
CanvasFrame canvas;
String title;
int delay;
/** Calls {@code this(canvas, -1)}. */
public ShowImageTransform(CanvasFrame canvas) {
this(canvas, -1);
}
/**
* Constructs an instance of the ImageTransform from a {@link CanvasFrame}.
*
* @param canvas to display images in
* @param delay max time to wait in milliseconds (0 == infinity, negative == no wait)
*/
public ShowImageTransform(CanvasFrame canvas, int delay) {
super(null);
this.canvas = canvas;
this.delay = delay;
}
/** Calls {@code this(title, -1)}. */
public ShowImageTransform(String title) {
this(title, -1);
}
/**
* Constructs an instance of the ImageTransform with a new {@link CanvasFrame}.
*
* @param title of the new CanvasFrame to display images in
* @param delay max time to wait in milliseconds (0 == infinity, negative == no wait)
*/
public ShowImageTransform(String title, int delay) {
super(null);
this.canvas = null;
this.title = title;
this.delay = delay;
}
@Override
protected ImageWritable doTransform(ImageWritable image, Random random) {
if (canvas == null) {
canvas = new CanvasFrame(title, 1.0);
canvas.setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE);
}
if (image == null) {
canvas.dispose();
return null;
}
if (!canvas.isVisible()) {
return image;
}
Frame frame = image.getFrame();
canvas.setCanvasSize(frame.imageWidth, frame.imageHeight);
canvas.showImage(frame);
if (delay >= 0) {
try {
canvas.waitKey(delay);
} catch (InterruptedException ex) {
// reset interrupt to be nice
Thread.currentThread().interrupt();
}
}
return image;
}
@Override
public float[] query(float... coordinates) {
return coordinates;
}
}
@@ -0,0 +1,146 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.transform;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
import org.bytedeco.javacpp.FloatPointer;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.datavec.image.data.ImageWritable;
import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties;
import org.nd4j.shade.jackson.annotation.JsonInclude;
import org.nd4j.shade.jackson.annotation.JsonProperty;
import java.nio.FloatBuffer;
import java.util.Random;
import org.bytedeco.opencv.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_imgproc.*;
@Accessors(fluent = true)
@JsonIgnoreProperties({"interMode", "borderMode", "borderValue", "converter"})
@JsonInclude(JsonInclude.Include.NON_NULL)
@Data
public class WarpImageTransform extends BaseImageTransform<Mat> {
private float[] deltas;
@Getter
@Setter
int interMode = INTER_LINEAR;
@Getter
@Setter
int borderMode = BORDER_CONSTANT;
@Getter
@Setter
Scalar borderValue = Scalar.ZERO;
private Mat M;
/** Calls {@code this(null, delta, delta, delta, delta, delta, delta, delta, delta)}. */
public WarpImageTransform(float delta) {
this(null, delta, delta, delta, delta, delta, delta, delta, delta);
}
/** Calls {@code this(random, delta, delta, delta, delta, delta, delta, delta, delta)}. */
public WarpImageTransform(Random random, float delta) {
this(random, delta, delta, delta, delta, delta, delta, delta, delta);
}
/** Calls {@code this(null, dx1, dy1, dx2, dy2, dx3, dy3, dx4, dy4)}. */
public WarpImageTransform(@JsonProperty("deltas[0]") float dx1, @JsonProperty("deltas[1]") float dy1,
@JsonProperty("deltas[2]") float dx2, @JsonProperty("deltas[3]") float dy2,
@JsonProperty("deltas[4]") float dx3, @JsonProperty("deltas[5]") float dy3,
@JsonProperty("deltas[6]") float dx4, @JsonProperty("deltas[7]") float dy4) {
this(null, dx1, dy1, dx2, dy2, dx3, dy3, dx4, dy4);
}
/**
* Constructs an instance of the ImageTransform.
*
* @param random object to use (or null for deterministic)
* @param dx1 maximum warping in x for the top-left corner (pixels)
* @param dy1 maximum warping in y for the top-left corner (pixels)
* @param dx2 maximum warping in x for the top-right corner (pixels)
* @param dy2 maximum warping in y for the top-right corner (pixels)
* @param dx3 maximum warping in x for the bottom-right corner (pixels)
* @param dy3 maximum warping in y for the bottom-right corner (pixels)
* @param dx4 maximum warping in x for the bottom-left corner (pixels)
* @param dy4 maximum warping in y for the bottom-left corner (pixels)
*/
public WarpImageTransform(Random random, float dx1, float dy1, float dx2, float dy2, float dx3, float dy3,
float dx4, float dy4) {
super(random);
deltas = new float[8];
deltas[0] = dx1;
deltas[1] = dy1;
deltas[2] = dx2;
deltas[3] = dy2;
deltas[4] = dx3;
deltas[5] = dy3;
deltas[6] = dx4;
deltas[7] = dy4;
this.converter = new OpenCVFrameConverter.ToMat();
}
/**
* Takes an image and returns a transformed image.
* Uses the random object in the case of random transformations.
*
* @param image to transform, null == end of stream
* @param random object to use (or null for deterministic)
* @return transformed image
*/
@Override
protected ImageWritable doTransform(ImageWritable image, Random random) {
if (image == null) {
return null;
}
Mat mat = converter.convert(image.getFrame());
Point2f src = new Point2f(4);
Point2f dst = new Point2f(4);
src.put(0, 0, mat.cols(), 0, mat.cols(), mat.rows(), 0, mat.rows());
for (int i = 0; i < 8; i++) {
dst.put(i, src.get(i) + deltas[i] * (random != null ? 2 * random.nextFloat() - 1 : 1));
}
Mat result = new Mat();
M = getPerspectiveTransform(src, dst);
warpPerspective(mat, result, M, mat.size(), interMode, borderMode, borderValue);
return new ImageWritable(converter.convert(result));
}
@Override
public float[] query(float... coordinates) {
Mat src = new Mat(1, coordinates.length / 2, CV_32FC2, new FloatPointer(coordinates));
Mat dst = new Mat();
perspectiveTransform(src, dst, M);
FloatBuffer buf = dst.createBuffer();
float[] transformed = new float[coordinates.length];
buf.get(transformed);
return transformed;
}
}
@@ -0,0 +1,44 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.datavec.image.util;
public class ImageUtils {
/**
* Calculate coordinates in an image, assuming the image has been scaled from (oH x oW) pixels to (nH x nW) pixels
*
* @param x X position (pixels) to translate
* @param y Y position (pixels) to translate
* @param origImageW Original image width (pixels)
* @param origImageH Original image height (pixels)
* @param newImageW New image width (pixels)
* @param newImageH New image height (pixels)
* @return New X and Y coordinates (pixels, in new image)
*/
public static double[] translateCoordsScaleImage(double x, double y, double origImageW, double origImageH, double newImageW, double newImageH){
double newX = x * newImageW / origImageW;
double newY = y * newImageH / origImageH;
return new double[]{newX, newY};
}
}
@@ -0,0 +1,31 @@
open module datavec.data.image {
requires commons.io;
requires guava;
requires imageio.bmp;
requires imageio.core;
requires imageio.jpeg;
requires imageio.psd;
requires jai.imageio.core;
requires org.bytedeco.ffmpeg;
requires org.bytedeco.hdf5;
requires org.bytedeco.javacpp;
requires org.bytedeco.leptonica;
requires resources;
requires slf4j.api;
requires static android;
requires datavec.api;
requires jackson;
requires java.desktop;
requires nd4j.api;
requires nd4j.common;
requires org.bytedeco.javacv;
requires org.bytedeco.opencv;
exports org.datavec.image.data;
exports org.datavec.image.format;
exports org.datavec.image.loader;
exports org.datavec.image.recordreader;
exports org.datavec.image.recordreader.objdetect;
exports org.datavec.image.recordreader.objdetect.impl;
exports org.datavec.image.transform;
exports org.datavec.image.util;
}
@@ -0,0 +1,98 @@
[
{
"name":"org.datavec.image.transform.ImageTransform",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true,
"allDeclaredClasses" : true,
"allPublicClasses" : true
},
{
"name":"org.datavec.image.transform.BoxImageTransform",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true,
"allDeclaredClasses" : true,
"allPublicClasses" : true
},
{
"name":"org.datavec.image.transform.ColorConversionTransform",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true,
"allDeclaredClasses" : true,
"allPublicClasses" : true
},
{
"name":"org.datavec.image.transform.CropImageTransform",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true,
"allDeclaredClasses" : true,
"allPublicClasses" : true
},
{
"name":"org.datavec.image.transform.EqualizeHistTransform",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true,
"allDeclaredClasses" : true,
"allPublicClasses" : true
},
{
"name":"org.datavec.image.transform.FilterImageTransform",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true,
"allDeclaredClasses" : true,
"allPublicClasses" : true
},
{
"name":"org.datavec.image.transform.FlipImageTransform",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true,
"allDeclaredClasses" : true,
"allPublicClasses" : true
},
{
"name":"org.datavec.image.transform.RotateImageTransform",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true,
"allDeclaredClasses" : true,
"allPublicClasses" : true
},
{
"name":"org.datavec.image.transform.ResizeImageTransform",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true,
"allDeclaredClasses" : true,
"allPublicClasses" : true
},
{
"name":"org.datavec.image.transform.RotateImageTransform",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true,
"allDeclaredClasses" : true,
"allPublicClasses" : true
},
{
"name":"org.datavec.image.transform.WarpImageTransform",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true,
"allDeclaredClasses" : true,
"allPublicClasses" : true
},
{
"name":"org.datavec.image.transform.RandomCropTransform",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true,
"allDeclaredClasses" : true,
"allPublicClasses" : true
}
]
@@ -0,0 +1,52 @@
<!--
~ /* ******************************************************************************
~ *
~ *
~ * This program and the accompanying materials are made available under the
~ * terms of the Apache License, Version 2.0 which is available at
~ * https://www.apache.org/licenses/LICENSE-2.0.
~ *
~ * See the NOTICE file distributed with this work for additional
~ * information regarding copyright ownership.
~ * Unless required by applicable law or agreed to in writing, software
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
~ * License for the specific language governing permissions and limitations
~ * under the License.
~ *
~ * SPDX-License-Identifier: Apache-2.0
~ ******************************************************************************/
-->
<configuration>
<appender name="FILE" class="ch.qos.logback.core.FileAppender">
<file>logs/application.log</file>
<encoder>
<pattern>%date - [%level] - from %logger in %thread
%n%message%n%xException%n</pattern>
</encoder>
</appender>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern> %logger{15} - %message%n%xException{5}
</pattern>
</encoder>
</appender>
<logger name="org.apache.catalina.core" level="DEBUG" />
<logger name="org.springframework" level="DEBUG" />
<logger name="org.datavec" level="DEBUG" />
<logger name="org.nd4j" level="INFO" />
<logger name="opennlp.uima.util" level="OFF" />
<logger name="org.apache.uima" level="OFF" />
<logger name="org.cleartk" level="OFF" />
<root level="ERROR">
<appender-ref ref="STDOUT" />
<appender-ref ref="FILE" />
</root>
</configuration>