chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
License Check / license-check (push) Failing after 2s

This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
+11
View File
@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="org.tensorflow.lite">
<!-- TFLite Java Library is built against NDK API 21. -->
<uses-sdk
android:minSdkVersion="21" />
<application />
</manifest>
@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="org.tensorflow.lite.api">
<!-- TFLite Java Library is built against NDK API 21. -->
<uses-sdk
android:minSdkVersion="21" />
<application />
</manifest>
@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="org.tensorflow.lite.gpu">
<!-- TFLite Java Library is built against NDK API 21. -->
<uses-sdk
android:minSdkVersion="21" />
<application />
</manifest>
@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="org.tensorflow.lite.gpu.api">
<uses-sdk
android:minSdkVersion="21" />
<application>
<!-- Applications that target Android S+ require explicit declaration of
any referenced vendor-provided libraries. -->
<uses-native-library
android:name="libOpenCL.so"
android:required="false" />
<uses-native-library
android:name="libOpenCL-car.so"
android:required="false" />
<uses-native-library
android:name="libOpenCL-pixel.so"
android:required="false" />
</application>
</manifest>
File diff suppressed because it is too large Load Diff
+154
View File
@@ -0,0 +1,154 @@
"""Generate zipped aar file including different variants of .so in jni folder."""
load("@build_bazel_rules_android//android:rules.bzl", "android_binary")
def aar_with_jni(
name,
android_library,
headers = None,
flatten_headers = False,
strip_headers_prefix = "",
license_file = "//:LICENSE",
third_party_notice = None):
"""Generates an Android AAR with repo root license given an Android library target.
Args:
name: Name of the generated .aar file.
android_library: The `android_library` target to package. Note that the
AAR will contain *only that library's .jar` sources. It does not
package the transitive closure of all Java source dependencies.
headers: Optional list of headers that will be included in the
generated .aar file. This is useful for distributing self-contained
.aars with native libs that can be used directly by native clients.
flatten_headers: Whether to flatten the output paths of included headers.
strip_headers_prefix: The prefix to strip from the output paths of included headers.
license_file: Optional. The main LICENSE file to include in the AAR.
Defaults to //third_party/tensorflow:LICENSE.
third_party_notice: Optional. The third party dependency licenses as THIRD_PARTY_NOTICE.txt.
"""
# Generate dummy AndroidManifest.xml for dummy apk usage
# (dummy apk is generated by <name>_dummy_app_for_so target below)
native.genrule(
name = name + "_binary_manifest_generator",
outs = [name + "_generated_AndroidManifest.xml"],
cmd = """
cat > $(OUTS) <<EOF
<manifest
xmlns:android="http://schemas.android.com/apk/res/android"
package="dummy.package.for.so">
<uses-sdk android:minSdkVersion="34"/>
</manifest>
EOF
""",
)
# Generate dummy apk including .so files and later we extract out
# .so files and throw away the apk.
android_binary(
name = name + "_dummy_app_for_so",
manifest = name + "_generated_AndroidManifest.xml",
custom_package = "dummy.package.for.so",
deps = [android_library],
# In some platforms we don't have an Android SDK/NDK and this target
# can't be built. We need to prevent the build system from trying to
# use the target in that case.
tags = [
"manual",
"no_cuda_on_cpu_tap",
],
)
srcs = [
android_library + ".aar",
name + "_dummy_app_for_so_unsigned.apk",
license_file,
]
cmd = """
cp $(location {0}.aar) $(location :{1}.aar)
chmod +w $(location :{1}.aar)
origdir=$$PWD
cd $$(mktemp -d)
unzip $$origdir/$(location :{1}_dummy_app_for_so_unsigned.apk) "lib/*"
cp -r lib jni
zip -r $$origdir/$(location :{1}.aar) jni/*/*.so
cp $$origdir/$(location {2}) ./LICENSE
zip $$origdir/$(location :{1}.aar) LICENSE
""".format(android_library, name, license_file)
if headers:
srcs += headers
cmd += """
mkdir headers
"""
for src in headers:
if flatten_headers:
cmd += """
cp -RL $$origdir/$(location {0}) headers/$$(basename $(location {0}))
""".format(src)
else:
cmd += """
default_dir=$$(dirname $(rootpath {0}))
modified_dir=$$(echo $$default_dir | sed -e 's/^{1}//g')
mkdir -p headers/$$modified_dir
cp -RL $$origdir/$(location {0}) headers/$$modified_dir
if [ -n "{1}" ]; then
sed -i -e 's/^#include \"{1}/#include \"/g' headers/$$modified_dir/$$(basename $(location {0}))
fi
""".format(src, strip_headers_prefix.replace("/", "\\/"))
cmd += "zip -r $$origdir/$(location :{0}.aar) headers".format(name)
if third_party_notice:
srcs.append(third_party_notice)
cmd += """
cp $$origdir/$(location {0}) ./THIRD_PARTY_NOTICE.txt
zip $$origdir/$(location :{1}.aar) THIRD_PARTY_NOTICE.txt
""".format(third_party_notice, name)
native.genrule(
name = name,
srcs = srcs,
outs = [name + ".aar"],
# In some platforms we don't have an Android SDK/NDK and this target
# can't be built. We need to prevent the build system from trying to
# use the target in that case.
tags = ["manual"],
cmd = cmd,
)
def aar_without_jni(
name,
android_library,
license_file = "//:LICENSE"):
"""Generates an Android AAR with repo root license given a pure Java Android library target.
Args:
name: Name of the generated .aar file.
android_library: The `android_library` target to package. Note that the
AAR will contain *only that library's .jar` sources. It does not
package the transitive closure of all Java source dependencies.
license_file: Optional. The main LICENSE file to include in the AAR.
Defaults to //third_party/tensorflow:LICENSE.
"""
srcs = [
android_library + ".aar",
license_file,
]
cmd = """
cp $(location {0}.aar) $(location :{1}.aar)
chmod +w $(location :{1}.aar)
origdir=$$PWD
cd $$(mktemp -d)
cp $$origdir/$(location {2}) ./LICENSE
zip $$origdir/$(location :{1}.aar) LICENSE
""".format(android_library, name, license_file)
native.genrule(
name = name,
srcs = srcs,
outs = [name + ".aar"],
cmd = cmd,
)
+66
View File
@@ -0,0 +1,66 @@
#!/bin/bash
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
set -e
set -x
TMPDIR=`mktemp -d`
trap "rm -rf $TMPDIR" EXIT
VERSION=1.0
BUILDER=bazel
BASEDIR=tensorflow/lite
CROSSTOOL="//external:android/crosstool"
HOST_CROSSTOOL="@bazel_tools//tools/cpp:toolchain"
BUILD_OPTS="-c opt"
CROSSTOOL_OPTS="--crosstool_top=$CROSSTOOL --host_crosstool_top=$HOST_CROSSTOOL"
test -d $BASEDIR || (echo "Aborting: not at top-level build directory"; exit 1)
function build_basic_aar() {
local OUTDIR=$1
$BUILDER build $BUILD_OPTS $BASEDIR/java:tensorflowlite.aar
unzip -d $OUTDIR $BUILDER-bin/$BASEDIR/java/tensorflowlite.aar
# targetSdkVersion is here to prevent the app from requesting spurious
# permissions, such as permission to make phone calls. It worked for v1.0,
# but minSdkVersion might be the preferred way to handle this.
sed -i -e 's/<application>/<uses-sdk android:targetSdkVersion="25"\/><application>/' $OUTDIR/AndroidManifest.xml
}
function build_arch() {
local ARCH=$1
local CONFIG=$2
local OUTDIR=$3
mkdir -p $OUTDIR/jni/$ARCH/
$BUILDER build $BUILD_OPTS $CROSSTOOL_OPTS --cpu=$CONFIG \
$BASEDIR/java:libtensorflowlite_jni.so
cp $BUILDER-bin/$BASEDIR/java/libtensorflowlite_jni.so $OUTDIR/jni/$ARCH/
}
rm -rf $TMPDIR
mkdir -p $TMPDIR/jni
build_basic_aar $TMPDIR
build_arch arm64-v8a arm64-v8a $TMPDIR
build_arch armeabi-v7a armeabi-v7a $TMPDIR
build_arch x86 x86 $TMPDIR
build_arch x86_64 x86_64 $TMPDIR
AAR_FILE=`realpath tflite-${VERSION}.aar`
(cd $TMPDIR && zip $AAR_FILE -r *)
echo "New AAR file is $AAR_FILE"
+29
View File
@@ -0,0 +1,29 @@
# This file is based on https://github.com/github/gitignore/blob/master/Android.gitignore
*.iml
.idea/compiler.xml
.idea/copyright
.idea/dictionaries
.idea/gradle.xml
.idea/libraries
.idea/inspectionProfiles
.idea/misc.xml
.idea/modules.xml
.idea/runConfigurations.xml
.idea/tasks.xml
.idea/workspace.xml
.gradle
local.properties
.DS_Store
build/
gradleBuild/
*.apk
*.ap_
*.dex
*.class
bin/
gen/
out/
*.log
.navigation/
/captures
.externalNativeBuild
+52
View File
@@ -0,0 +1,52 @@
# TF Lite Android Image Classifier App Example
A simple Android example that demonstrates image classification using the camera.
## Building in Android Studio with TensorFlow Lite AAR from MavenCentral.
The build.gradle is configured to use TensorFlow Lite's nightly build.
If you see a build error related to compatibility with Tensorflow Lite's Java API (example: method X is
undefined for type Interpreter), there has likely been a backwards compatible
change to the API. You will need to pull new app code that's compatible with the
nightly build and may need to first wait a few days for our external and internal
code to merge.
## Building from Source with Bazel
1. Follow the [Bazel steps for the TF Demo App](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#bazel):
1. [Install Bazel and Android Prerequisites](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#install-bazel-and-android-prerequisites).
It's easiest with Android Studio.
- You'll need at least SDK version 23.
- Make sure to install the latest version of Bazel. Some distributions
ship with Bazel 0.5.4, which is too old.
- Bazel requires Android Build Tools `28.0.0` or higher.
- You also need to install the Android Support Repository, available
through Android Studio under `Android SDK Manager -> SDK Tools ->
Android Support Repository`.
2. [Edit your `WORKSPACE`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#edit-workspace)
to add SDK and NDK targets.
NOTE: As long as you have the SDK and NDK installed, the `./configure`
script will create these rules for you. Answer "Yes" when the script asks
to automatically configure the `./WORKSPACE`.
- Make sure the `api_level` in `WORKSPACE` is set to an SDK version that
you have installed.
- By default, Android Studio will install the SDK to `~/Android/Sdk` and
the NDK to `~/Android/Sdk/ndk-bundle`.
2. Build the app with Bazel. The demo needs C++11:
```shell
bazel build -c opt //tensorflow/lite/java/demo/app/src/main:TfLiteCameraDemo
```
3. Install the demo on a
[debug-enabled device](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#install):
```shell
adb install bazel-bin/tensorflow/lite/java/demo/app/src/main/TfLiteCameraDemo.apk
```
+135
View File
@@ -0,0 +1,135 @@
apply plugin: 'com.android.application'
android {
compileSdkVersion 26
buildToolsVersion "27.0.3"
defaultConfig {
applicationId "android.example.com.tflitecamerademo"
// Required by Camera2 API.
minSdkVersion 21
targetSdkVersion 26
versionCode 1
versionName "1.0"
}
lintOptions {
abortOnError false
}
buildTypes {
release {
minifyEnabled false
proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
}
}
aaptOptions {
noCompress "tflite"
}
compileOptions {
sourceCompatibility JavaVersion.VERSION_1_8
targetCompatibility JavaVersion.VERSION_1_8
}
}
repositories {
mavenCentral()
maven { // Only for snapshot artifacts
name 'ossrh-snapshot'
url 'https://oss.sonatype.org/content/repositories/snapshots'
}
}
allprojects {
repositories {
// Uncomment if you want to use a local repo.
// mavenLocal()
mavenCentral()
maven { // Only for snapshot artifacts
name 'ossrh-snapshot'
url 'http://oss.sonatype.org/content/repositories/snapshots'
}
}
}
dependencies {
implementation fileTree(dir: 'libs', include: ['*.jar'])
implementation 'com.android.support:appcompat-v7:25.2.0'
implementation 'com.android.support.constraint:constraint-layout:1.0.2'
implementation 'com.android.support:design:25.2.0'
implementation 'com.android.support:support-annotations:25.3.1'
implementation 'com.android.support:support-v13:25.2.0'
// Build off of nightly TensorFlow Lite
implementation 'org.tensorflow:tensorflow-lite:0.0.0-nightly-SNAPSHOT'
implementation 'org.tensorflow:tensorflow-lite-gpu:0.0.0-nightly-SNAPSHOT'
// Use local TensorFlow library
// implementation 'org.tensorflow:tensorflow-lite-local:0.0.0'
}
def targetFolder = "src/main/assets"
def modelFloatDownloadUrl = "https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_1.0_224.tgz"
def modelQuantDownloadUrl = "https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224_quant.tgz"
def localCacheFloat = "build/intermediates/mobilenet_v1_1.0_224.tgz"
def localCacheQuant = "build/intermediates/mmobilenet_v1_1.0_224_quant.tgz"
task downloadModelFloat(type: DownloadUrlTask) {
doFirst {
println "Downloading ${modelFloatDownloadUrl}"
}
sourceUrl = "${modelFloatDownloadUrl}"
target = file("${localCacheFloat}")
}
task downloadModelQuant(type: DownloadUrlTask) {
doFirst {
println "Downloading ${modelQuantDownloadUrl}"
}
sourceUrl = "${modelQuantDownloadUrl}"
target = file("${localCacheQuant}")
}
task unzipModelFloat(type: Copy, dependsOn: 'downloadModelFloat') {
doFirst {
println "Unzipping ${localCacheFloat}"
}
from tarTree("${localCacheFloat}")
into "${targetFolder}"
}
task unzipModelQuant(type: Copy, dependsOn: 'downloadModelQuant') {
doFirst {
println "Unzipping ${localCacheQuant}"
}
from tarTree("${localCacheQuant}")
into "${targetFolder}"
}
task cleanUnusedFiles(type: Delete, dependsOn: ['unzipModelFloat', 'unzipModelQuant']) {
delete fileTree("${targetFolder}").matching {
include "*.pb"
include "*.ckpt.*"
include "*.pbtxt.*"
include "*.quant_info.*"
include "*.meta"
}
}
// Ensure the model file is downloaded and extracted before every build
preBuild.dependsOn unzipModelFloat
preBuild.dependsOn unzipModelQuant
preBuild.dependsOn cleanUnusedFiles
class DownloadUrlTask extends DefaultTask {
@Input
String sourceUrl
@OutputFile
File target
@TaskAction
void download() {
ant.get(src: sourceUrl, dest: target)
}
}
@@ -0,0 +1,43 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright 2014 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.example.android.tflitecamerademo">
<uses-permission android:name="android.permission.CAMERA" />
<uses-feature android:name="android.hardware.camera" />
<uses-feature android:name="android.hardware.camera.autofocus" />
<uses-sdk android:minSdkVersion="21" />
<application android:allowBackup="true"
android:label="@string/app_name"
android:icon="@drawable/ic_launcher"
android:theme="@style/MaterialTheme">
<activity android:name="com.example.android.tflitecamerademo.CameraActivity"
android:screenOrientation="portrait"
android:label="@string/app_name">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity>
</application>
</manifest>
@@ -0,0 +1,33 @@
load("@build_bazel_rules_android//android:rules.bzl", "android_binary")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
default_visibility = ["//visibility:private"],
licenses = ["notice"],
)
android_binary(
name = "TfLiteCameraDemo",
srcs = glob(["java/**/*.java"]),
assets = [
"//tensorflow/lite/java/demo/app/src/main/assets:labels_mobilenet_quant_v1_224.txt",
"@tflite_mobilenet_quant//:mobilenet_v1_1.0_224_quant.tflite",
"@tflite_mobilenet_float//:mobilenet_v1_1.0_224.tflite",
],
assets_dir = "",
custom_package = "com.example.android.tflitecamerademo",
manifest = "AndroidManifest.xml",
nocompress_extensions = [
".tflite",
],
resource_files = glob(["res/**"]),
# In some platforms we don't have an Android SDK/NDK and this target
# can't be built. We need to prevent the build system from trying to
# use the target in that case.
tags = ["manual"],
deps = [
"//tensorflow/lite/java:tensorflowlite",
"//tensorflow/lite/java:tensorflowlite_gpu",
"//tensorflow/lite/java/src/testhelper/java/org/tensorflow/lite:testhelper",
],
)
@@ -0,0 +1,14 @@
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
default_visibility = ["//visibility:private"],
licenses = ["notice"],
)
exports_files(
glob(
["**/*"],
exclude = [
"BUILD",
],
),
)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,72 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package com.example.android.tflitecamerademo;
import android.content.Context;
import android.util.AttributeSet;
import android.view.TextureView;
/** A {@link TextureView} that can be adjusted to a specified aspect ratio. */
public class AutoFitTextureView extends TextureView {
private int mRatioWidth = 0;
private int mRatioHeight = 0;
public AutoFitTextureView(Context context) {
this(context, null);
}
public AutoFitTextureView(Context context, AttributeSet attrs) {
this(context, attrs, 0);
}
public AutoFitTextureView(Context context, AttributeSet attrs, int defStyle) {
super(context, attrs, defStyle);
}
/**
* Sets the aspect ratio for this view. The size of the view will be measured based on the ratio
* calculated from the parameters. Note that the actual sizes of parameters don't matter, that is,
* calling setAspectRatio(2, 3) and setAspectRatio(4, 6) make the same result.
*
* @param width Relative horizontal size
* @param height Relative vertical size
*/
public void setAspectRatio(int width, int height) {
if (width < 0 || height < 0) {
throw new IllegalArgumentException("Size cannot be negative.");
}
mRatioWidth = width;
mRatioHeight = height;
requestLayout();
}
@Override
protected void onMeasure(int widthMeasureSpec, int heightMeasureSpec) {
super.onMeasure(widthMeasureSpec, heightMeasureSpec);
int width = MeasureSpec.getSize(widthMeasureSpec);
int height = MeasureSpec.getSize(heightMeasureSpec);
if (0 == mRatioWidth || 0 == mRatioHeight) {
setMeasuredDimension(width, height);
} else {
if (width < height * mRatioWidth / mRatioHeight) {
setMeasuredDimension(width, width * mRatioHeight / mRatioWidth);
} else {
setMeasuredDimension(height * mRatioWidth / mRatioHeight, height);
}
}
}
}
@@ -0,0 +1,862 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package com.example.android.tflitecamerademo;
import android.app.Activity;
import android.app.AlertDialog;
import android.app.Dialog;
import android.app.DialogFragment;
import android.app.Fragment;
import android.content.Context;
import android.content.DialogInterface;
import android.content.pm.PackageInfo;
import android.content.pm.PackageManager;
import android.content.res.Configuration;
import android.graphics.Bitmap;
import android.graphics.ImageFormat;
import android.graphics.Matrix;
import android.graphics.Point;
import android.graphics.RectF;
import android.graphics.SurfaceTexture;
import android.hardware.camera2.CameraAccessException;
import android.hardware.camera2.CameraCaptureSession;
import android.hardware.camera2.CameraCharacteristics;
import android.hardware.camera2.CameraDevice;
import android.hardware.camera2.CameraManager;
import android.hardware.camera2.CaptureRequest;
import android.hardware.camera2.CaptureResult;
import android.hardware.camera2.TotalCaptureResult;
import android.hardware.camera2.params.StreamConfigurationMap;
import android.media.ImageReader;
import android.os.Bundle;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.Process;
import android.text.SpannableString;
import android.text.SpannableStringBuilder;
import android.util.Log;
import android.util.Size;
import android.view.LayoutInflater;
import android.view.Surface;
import android.view.TextureView;
import android.view.View;
import android.view.ViewGroup;
import android.widget.AdapterView;
import android.widget.ArrayAdapter;
import android.widget.ListView;
import android.widget.NumberPicker;
import android.widget.TextView;
import android.widget.Toast;
import androidx.annotation.NonNull;
import android.support.v13.app.FragmentCompat;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
/** Basic fragments for the Camera. */
public class Camera2BasicFragment extends Fragment
implements FragmentCompat.OnRequestPermissionsResultCallback {
/** Tag for the {@link Log}. */
private static final String TAG = "TfLiteCameraDemo";
private static final String FRAGMENT_DIALOG = "dialog";
private static final String HANDLE_THREAD_NAME = "CameraBackground";
private static final int PERMISSIONS_REQUEST_CODE = 1;
private final Object lock = new Object();
private boolean runClassifier = false;
private boolean checkedPermissions = false;
private TextView textView;
private NumberPicker np;
private ImageClassifier classifier;
private ListView deviceView;
private ListView modelView;
/** Max preview width that is guaranteed by Camera2 API */
private static final int MAX_PREVIEW_WIDTH = 1920;
/** Max preview height that is guaranteed by Camera2 API */
private static final int MAX_PREVIEW_HEIGHT = 1080;
/**
* {@link TextureView.SurfaceTextureListener} handles several lifecycle events on a {@link
* TextureView}.
*/
private final TextureView.SurfaceTextureListener surfaceTextureListener =
new TextureView.SurfaceTextureListener() {
@Override
public void onSurfaceTextureAvailable(SurfaceTexture texture, int width, int height) {
openCamera(width, height);
}
@Override
public void onSurfaceTextureSizeChanged(SurfaceTexture texture, int width, int height) {
configureTransform(width, height);
}
@Override
public boolean onSurfaceTextureDestroyed(SurfaceTexture texture) {
return true;
}
@Override
public void onSurfaceTextureUpdated(SurfaceTexture texture) {}
};
// Model parameter constants.
private String gpu;
private String cpu;
private String nnApi;
private String mobilenetV1Quant;
private String mobilenetV1Float;
/** ID of the current {@link CameraDevice}. */
private String cameraId;
/** An {@link AutoFitTextureView} for camera preview. */
private AutoFitTextureView textureView;
/** A {@link CameraCaptureSession } for camera preview. */
private CameraCaptureSession captureSession;
/** A reference to the opened {@link CameraDevice}. */
private CameraDevice cameraDevice;
/** The {@link android.util.Size} of camera preview. */
private Size previewSize;
/** {@link CameraDevice.StateCallback} is called when {@link CameraDevice} changes its state. */
private final CameraDevice.StateCallback stateCallback =
new CameraDevice.StateCallback() {
@Override
public void onOpened(@NonNull CameraDevice currentCameraDevice) {
// This method is called when the camera is opened. We start camera preview here.
cameraOpenCloseLock.release();
cameraDevice = currentCameraDevice;
createCameraPreviewSession();
}
@Override
public void onDisconnected(@NonNull CameraDevice currentCameraDevice) {
cameraOpenCloseLock.release();
currentCameraDevice.close();
cameraDevice = null;
}
@Override
public void onError(@NonNull CameraDevice currentCameraDevice, int error) {
cameraOpenCloseLock.release();
currentCameraDevice.close();
cameraDevice = null;
Activity activity = getActivity();
if (null != activity) {
activity.finish();
}
}
};
private ArrayList<String> deviceStrings = new ArrayList<String>();
private ArrayList<String> modelStrings = new ArrayList<String>();
/** Current indices of device and model. */
int currentDevice = -1;
int currentModel = -1;
int currentNumThreads = -1;
/** An additional thread for running tasks that shouldn't block the UI. */
private HandlerThread backgroundThread;
/** A {@link Handler} for running tasks in the background. */
private Handler backgroundHandler;
/** An {@link ImageReader} that handles image capture. */
private ImageReader imageReader;
/** {@link CaptureRequest.Builder} for the camera preview */
private CaptureRequest.Builder previewRequestBuilder;
/** {@link CaptureRequest} generated by {@link #previewRequestBuilder} */
private CaptureRequest previewRequest;
/** A {@link Semaphore} to prevent the app from exiting before closing the camera. */
private Semaphore cameraOpenCloseLock = new Semaphore(1);
/** A {@link CameraCaptureSession.CaptureCallback} that handles events related to capture. */
private CameraCaptureSession.CaptureCallback captureCallback =
new CameraCaptureSession.CaptureCallback() {
@Override
public void onCaptureProgressed(
@NonNull CameraCaptureSession session,
@NonNull CaptureRequest request,
@NonNull CaptureResult partialResult) {}
@Override
public void onCaptureCompleted(
@NonNull CameraCaptureSession session,
@NonNull CaptureRequest request,
@NonNull TotalCaptureResult result) {}
};
/**
* Shows a {@link Toast} on the UI thread for the classification results.
*
* @param text The message to show
*/
private void showToast(String s) {
SpannableStringBuilder builder = new SpannableStringBuilder();
SpannableString str1 = new SpannableString(s);
builder.append(str1);
showToast(builder);
}
private void showToast(SpannableStringBuilder builder) {
final Activity activity = getActivity();
if (activity != null) {
activity.runOnUiThread(
new Runnable() {
@Override
public void run() {
textView.setText(builder, TextView.BufferType.SPANNABLE);
}
});
}
}
/**
* Resizes image.
*
* Attempting to use too large a preview size could exceed the camera bus' bandwidth limitation,
* resulting in gorgeous previews but the storage of garbage capture data.
*
* Given {@code choices} of {@code Size}s supported by a camera, choose the smallest one that is
* at least as large as the respective texture view size, and that is at most as large as the
* respective max size, and whose aspect ratio matches with the specified value. If such size
* doesn't exist, choose the largest one that is at most as large as the respective max size, and
* whose aspect ratio matches with the specified value.
*
* @param choices The list of sizes that the camera supports for the intended output class
* @param textureViewWidth The width of the texture view relative to sensor coordinate
* @param textureViewHeight The height of the texture view relative to sensor coordinate
* @param maxWidth The maximum width that can be chosen
* @param maxHeight The maximum height that can be chosen
* @param aspectRatio The aspect ratio
* @return The optimal {@code Size}, or an arbitrary one if none were big enough
*/
private static Size chooseOptimalSize(
Size[] choices,
int textureViewWidth,
int textureViewHeight,
int maxWidth,
int maxHeight,
Size aspectRatio) {
// Collect the supported resolutions that are at least as big as the preview Surface
List<Size> bigEnough = new ArrayList<>();
// Collect the supported resolutions that are smaller than the preview Surface
List<Size> notBigEnough = new ArrayList<>();
int w = aspectRatio.getWidth();
int h = aspectRatio.getHeight();
for (Size option : choices) {
if (option.getWidth() <= maxWidth
&& option.getHeight() <= maxHeight
&& option.getHeight() == option.getWidth() * h / w) {
if (option.getWidth() >= textureViewWidth && option.getHeight() >= textureViewHeight) {
bigEnough.add(option);
} else {
notBigEnough.add(option);
}
}
}
// Pick the smallest of those big enough. If there is no one big enough, pick the
// largest of those not big enough.
if (bigEnough.size() > 0) {
return Collections.min(bigEnough, new CompareSizesByArea());
} else if (notBigEnough.size() > 0) {
return Collections.max(notBigEnough, new CompareSizesByArea());
} else {
Log.e(TAG, "Couldn't find any suitable preview size");
return choices[0];
}
}
public static Camera2BasicFragment newInstance() {
return new Camera2BasicFragment();
}
/** Layout the preview and buttons. */
@Override
public View onCreateView(
LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) {
return inflater.inflate(R.layout.fragment_camera2_basic, container, false);
}
private void updateActiveModel() {
// Get UI information before delegating to background
final int modelIndex = modelView.getCheckedItemPosition();
final int deviceIndex = deviceView.getCheckedItemPosition();
final int numThreads = np.getValue();
backgroundHandler.post(
() -> {
if (modelIndex == currentModel
&& deviceIndex == currentDevice
&& numThreads == currentNumThreads) {
return;
}
currentModel = modelIndex;
currentDevice = deviceIndex;
currentNumThreads = numThreads;
// Disable classifier while updating
if (classifier != null) {
classifier.close();
classifier = null;
}
// Lookup names of parameters.
String model = modelStrings.get(modelIndex);
String device = deviceStrings.get(deviceIndex);
Log.i(TAG, "Changing model to " + model + " device " + device);
// Try to load model.
try {
if (model.equals(mobilenetV1Quant)) {
classifier = new ImageClassifierQuantizedMobileNet(getActivity());
} else if (model.equals(mobilenetV1Float)) {
classifier = new ImageClassifierFloatMobileNet(getActivity());
} else {
showToast("Failed to load model");
}
} catch (IOException e) {
Log.d(TAG, "Failed to load", e);
classifier = null;
}
// Customize the interpreter to the type of device we want to use.
if (classifier == null) {
return;
}
classifier.setNumThreads(numThreads);
if (device.equals(cpu)) {
} else if (device.equals(gpu)) {
classifier.useGpu();
} else if (device.equals(nnApi)) {
classifier.useNNAPI();
}
});
}
/** Connect the buttons to their event handler. */
@Override
public void onViewCreated(final View view, Bundle savedInstanceState) {
gpu = getString(R.string.gpu);
cpu = getString(R.string.cpu);
nnApi = getString(R.string.nnapi);
mobilenetV1Quant = getString(R.string.mobilenetV1Quant);
mobilenetV1Float = getString(R.string.mobilenetV1Float);
// Get references to widgets.
textureView = (AutoFitTextureView) view.findViewById(R.id.texture);
textView = (TextView) view.findViewById(R.id.text);
deviceView = (ListView) view.findViewById(R.id.device);
modelView = (ListView) view.findViewById(R.id.model);
// Build list of models
modelStrings.add(mobilenetV1Quant);
modelStrings.add(mobilenetV1Float);
// Build list of devices
int defaultModelIndex = 0;
deviceStrings.add(cpu);
deviceStrings.add(gpu);
deviceStrings.add(nnApi);
deviceView.setAdapter(
new ArrayAdapter<String>(
getContext(), R.layout.listview_row, R.id.listview_row_text, deviceStrings));
deviceView.setChoiceMode(ListView.CHOICE_MODE_SINGLE);
deviceView.setOnItemClickListener(
new AdapterView.OnItemClickListener() {
@Override
public void onItemClick(AdapterView<?> parent, View view, int position, long id) {
updateActiveModel();
}
});
deviceView.setItemChecked(0, true);
modelView.setChoiceMode(ListView.CHOICE_MODE_SINGLE);
ArrayAdapter<String> modelAdapter =
new ArrayAdapter<>(
getContext(), R.layout.listview_row, R.id.listview_row_text, modelStrings);
modelView.setAdapter(modelAdapter);
modelView.setItemChecked(defaultModelIndex, true);
modelView.setOnItemClickListener(
new AdapterView.OnItemClickListener() {
@Override
public void onItemClick(AdapterView<?> parent, View view, int position, long id) {
updateActiveModel();
}
});
np = (NumberPicker) view.findViewById(R.id.np);
np.setMinValue(1);
np.setMaxValue(10);
np.setWrapSelectorWheel(true);
np.setOnValueChangedListener(
new NumberPicker.OnValueChangeListener() {
@Override
public void onValueChange(NumberPicker picker, int oldVal, int newVal) {
updateActiveModel();
}
});
// Start initial model.
}
/** Load the model and labels. */
@Override
public void onActivityCreated(Bundle savedInstanceState) {
super.onActivityCreated(savedInstanceState);
startBackgroundThread();
}
@Override
public void onResume() {
super.onResume();
startBackgroundThread();
// When the screen is turned off and turned back on, the SurfaceTexture is already
// available, and "onSurfaceTextureAvailable" will not be called. In that case, we can open
// a camera and start preview from here (otherwise, we wait until the surface is ready in
// the SurfaceTextureListener).
if (textureView.isAvailable()) {
openCamera(textureView.getWidth(), textureView.getHeight());
} else {
textureView.setSurfaceTextureListener(surfaceTextureListener);
}
}
@Override
public void onPause() {
closeCamera();
stopBackgroundThread();
super.onPause();
}
@Override
public void onDestroy() {
if (classifier != null) {
classifier.close();
}
super.onDestroy();
}
/**
* Sets up member variables related to camera.
*
* @param width The width of available size for camera preview
* @param height The height of available size for camera preview
*/
private void setUpCameraOutputs(int width, int height) {
Activity activity = getActivity();
CameraManager manager = (CameraManager) activity.getSystemService(Context.CAMERA_SERVICE);
try {
for (String cameraId : manager.getCameraIdList()) {
CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId);
// We don't use a front facing camera in this sample.
Integer facing = characteristics.get(CameraCharacteristics.LENS_FACING);
if (facing != null && facing == CameraCharacteristics.LENS_FACING_FRONT) {
continue;
}
StreamConfigurationMap map =
characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
if (map == null) {
continue;
}
// // For still image captures, we use the largest available size.
Size largest =
Collections.max(
Arrays.asList(map.getOutputSizes(ImageFormat.JPEG)), new CompareSizesByArea());
imageReader =
ImageReader.newInstance(
largest.getWidth(), largest.getHeight(), ImageFormat.JPEG, /*maxImages*/ 2);
// Find out if we need to swap dimension to get the preview size relative to sensor
// coordinate.
int displayRotation = activity.getWindowManager().getDefaultDisplay().getRotation();
// noinspection ConstantConditions
/* Orientation of the camera sensor */
int sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION);
boolean swappedDimensions = false;
switch (displayRotation) {
case Surface.ROTATION_0:
case Surface.ROTATION_180:
if (sensorOrientation == 90 || sensorOrientation == 270) {
swappedDimensions = true;
}
break;
case Surface.ROTATION_90:
case Surface.ROTATION_270:
if (sensorOrientation == 0 || sensorOrientation == 180) {
swappedDimensions = true;
}
break;
default:
Log.e(TAG, "Display rotation is invalid: " + displayRotation);
}
Point displaySize = new Point();
activity.getWindowManager().getDefaultDisplay().getSize(displaySize);
int rotatedPreviewWidth = width;
int rotatedPreviewHeight = height;
int maxPreviewWidth = displaySize.x;
int maxPreviewHeight = displaySize.y;
if (swappedDimensions) {
rotatedPreviewWidth = height;
rotatedPreviewHeight = width;
maxPreviewWidth = displaySize.y;
maxPreviewHeight = displaySize.x;
}
if (maxPreviewWidth > MAX_PREVIEW_WIDTH) {
maxPreviewWidth = MAX_PREVIEW_WIDTH;
}
if (maxPreviewHeight > MAX_PREVIEW_HEIGHT) {
maxPreviewHeight = MAX_PREVIEW_HEIGHT;
}
previewSize =
chooseOptimalSize(
map.getOutputSizes(SurfaceTexture.class),
rotatedPreviewWidth,
rotatedPreviewHeight,
maxPreviewWidth,
maxPreviewHeight,
largest);
// We fit the aspect ratio of TextureView to the size of preview we picked.
int orientation = getResources().getConfiguration().orientation;
if (orientation == Configuration.ORIENTATION_LANDSCAPE) {
textureView.setAspectRatio(previewSize.getWidth(), previewSize.getHeight());
} else {
textureView.setAspectRatio(previewSize.getHeight(), previewSize.getWidth());
}
this.cameraId = cameraId;
return;
}
} catch (CameraAccessException e) {
Log.e(TAG, "Failed to access Camera", e);
} catch (NullPointerException e) {
// Currently an NPE is thrown when the Camera2API is used but not supported on the
// device this code runs.
ErrorDialog.newInstance(getString(R.string.camera_error))
.show(getChildFragmentManager(), FRAGMENT_DIALOG);
}
}
private String[] getRequiredPermissions() {
Activity activity = getActivity();
try {
PackageInfo info =
activity
.getPackageManager()
.getPackageInfo(activity.getPackageName(), PackageManager.GET_PERMISSIONS);
String[] ps = info.requestedPermissions;
if (ps != null && ps.length > 0) {
return ps;
} else {
return new String[0];
}
} catch (Exception e) {
return new String[0];
}
}
/** Opens the camera specified by {@link Camera2BasicFragment#cameraId}. */
private void openCamera(int width, int height) {
if (!checkedPermissions && !allPermissionsGranted()) {
FragmentCompat.requestPermissions(this, getRequiredPermissions(), PERMISSIONS_REQUEST_CODE);
return;
} else {
checkedPermissions = true;
}
setUpCameraOutputs(width, height);
configureTransform(width, height);
Activity activity = getActivity();
CameraManager manager = (CameraManager) activity.getSystemService(Context.CAMERA_SERVICE);
try {
if (!cameraOpenCloseLock.tryAcquire(2500, TimeUnit.MILLISECONDS)) {
throw new RuntimeException("Time out waiting to lock camera opening.");
}
manager.openCamera(cameraId, stateCallback, backgroundHandler);
} catch (CameraAccessException e) {
Log.e(TAG, "Failed to open Camera", e);
} catch (InterruptedException e) {
throw new RuntimeException("Interrupted while trying to lock camera opening.", e);
}
}
private boolean allPermissionsGranted() {
for (String permission : getRequiredPermissions()) {
if (getActivity().checkPermission(permission, Process.myPid(), Process.myUid())
!= PackageManager.PERMISSION_GRANTED) {
return false;
}
}
return true;
}
@Override
public void onRequestPermissionsResult(
int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
super.onRequestPermissionsResult(requestCode, permissions, grantResults);
}
/** Closes the current {@link CameraDevice}. */
private void closeCamera() {
try {
cameraOpenCloseLock.acquire();
if (null != captureSession) {
captureSession.close();
captureSession = null;
}
if (null != cameraDevice) {
cameraDevice.close();
cameraDevice = null;
}
if (null != imageReader) {
imageReader.close();
imageReader = null;
}
} catch (InterruptedException e) {
throw new RuntimeException("Interrupted while trying to lock camera closing.", e);
} finally {
cameraOpenCloseLock.release();
}
}
/** Starts a background thread and its {@link Handler}. */
private void startBackgroundThread() {
backgroundThread = new HandlerThread(HANDLE_THREAD_NAME);
backgroundThread.start();
backgroundHandler = new Handler(backgroundThread.getLooper());
// Start the classification train & load an initial model.
synchronized (lock) {
runClassifier = true;
}
backgroundHandler.post(periodicClassify);
updateActiveModel();
}
/** Stops the background thread and its {@link Handler}. */
private void stopBackgroundThread() {
backgroundThread.quitSafely();
try {
backgroundThread.join();
backgroundThread = null;
backgroundHandler = null;
synchronized (lock) {
runClassifier = false;
}
} catch (InterruptedException e) {
Log.e(TAG, "Interrupted when stopping background thread", e);
}
}
/** Takes photos and classify them periodically. */
private Runnable periodicClassify =
new Runnable() {
@Override
public void run() {
synchronized (lock) {
if (runClassifier) {
classifyFrame();
}
}
backgroundHandler.post(periodicClassify);
}
};
/** Creates a new {@link CameraCaptureSession} for camera preview. */
private void createCameraPreviewSession() {
try {
SurfaceTexture texture = textureView.getSurfaceTexture();
assert texture != null;
// We configure the size of default buffer to be the size of camera preview we want.
texture.setDefaultBufferSize(previewSize.getWidth(), previewSize.getHeight());
// This is the output Surface we need to start preview.
Surface surface = new Surface(texture);
// We set up a CaptureRequest.Builder with the output Surface.
previewRequestBuilder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
previewRequestBuilder.addTarget(surface);
// Here, we create a CameraCaptureSession for camera preview.
cameraDevice.createCaptureSession(
Arrays.asList(surface),
new CameraCaptureSession.StateCallback() {
@Override
public void onConfigured(@NonNull CameraCaptureSession cameraCaptureSession) {
// The camera is already closed
if (null == cameraDevice) {
return;
}
// When the session is ready, we start displaying the preview.
captureSession = cameraCaptureSession;
try {
// Auto focus should be continuous for camera preview.
previewRequestBuilder.set(
CaptureRequest.CONTROL_AF_MODE,
CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE);
// Finally, we start displaying the camera preview.
previewRequest = previewRequestBuilder.build();
captureSession.setRepeatingRequest(
previewRequest, captureCallback, backgroundHandler);
} catch (CameraAccessException e) {
Log.e(TAG, "Failed to set up config to capture Camera", e);
}
}
@Override
public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession) {
showToast("Failed");
}
},
null);
} catch (CameraAccessException e) {
Log.e(TAG, "Failed to preview Camera", e);
}
}
/**
* Configures the necessary {@link android.graphics.Matrix} transformation to `textureView`. This
* method should be called after the camera preview size is determined in setUpCameraOutputs and
* also the size of `textureView` is fixed.
*
* @param viewWidth The width of `textureView`
* @param viewHeight The height of `textureView`
*/
private void configureTransform(int viewWidth, int viewHeight) {
Activity activity = getActivity();
if (null == textureView || null == previewSize || null == activity) {
return;
}
int rotation = activity.getWindowManager().getDefaultDisplay().getRotation();
Matrix matrix = new Matrix();
RectF viewRect = new RectF(0, 0, viewWidth, viewHeight);
RectF bufferRect = new RectF(0, 0, previewSize.getHeight(), previewSize.getWidth());
float centerX = viewRect.centerX();
float centerY = viewRect.centerY();
if (Surface.ROTATION_90 == rotation || Surface.ROTATION_270 == rotation) {
bufferRect.offset(centerX - bufferRect.centerX(), centerY - bufferRect.centerY());
matrix.setRectToRect(viewRect, bufferRect, Matrix.ScaleToFit.FILL);
float scale =
Math.max(
(float) viewHeight / previewSize.getHeight(),
(float) viewWidth / previewSize.getWidth());
matrix.postScale(scale, scale, centerX, centerY);
matrix.postRotate(90 * (rotation - 2), centerX, centerY);
} else if (Surface.ROTATION_180 == rotation) {
matrix.postRotate(180, centerX, centerY);
}
textureView.setTransform(matrix);
}
/** Classifies a frame from the preview stream. */
private void classifyFrame() {
if (classifier == null || getActivity() == null || cameraDevice == null) {
// It's important to not call showToast every frame, or else the app will starve and
// hang. updateActiveModel() already puts an error message up with showToast.
// showToast("Uninitialized Classifier or invalid context.");
return;
}
SpannableStringBuilder textToShow = new SpannableStringBuilder();
Bitmap bitmap = textureView.getBitmap(classifier.getImageSizeX(), classifier.getImageSizeY());
classifier.classifyFrame(bitmap, textToShow);
bitmap.recycle();
showToast(textToShow);
}
/** Compares two {@code Size}s based on their areas. */
private static class CompareSizesByArea implements Comparator<Size> {
@Override
public int compare(Size lhs, Size rhs) {
// We cast here to ensure the multiplications won't overflow
return Long.signum(
(long) lhs.getWidth() * lhs.getHeight() - (long) rhs.getWidth() * rhs.getHeight());
}
}
/** Shows an error message dialog. */
public static class ErrorDialog extends DialogFragment {
private static final String ARG_MESSAGE = "message";
public static ErrorDialog newInstance(String message) {
ErrorDialog dialog = new ErrorDialog();
Bundle args = new Bundle();
args.putString(ARG_MESSAGE, message);
dialog.setArguments(args);
return dialog;
}
@Override
public Dialog onCreateDialog(Bundle savedInstanceState) {
final Activity activity = getActivity();
return new AlertDialog.Builder(activity)
.setMessage(getArguments().getString(ARG_MESSAGE))
.setPositiveButton(
android.R.string.ok,
new DialogInterface.OnClickListener() {
@Override
public void onClick(DialogInterface dialogInterface, int i) {
activity.finish();
}
})
.create();
}
}
}
@@ -0,0 +1,35 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package com.example.android.tflitecamerademo;
import android.app.Activity;
import android.os.Bundle;
/** Main {@code Activity} class for the Camera app. */
public class CameraActivity extends Activity {
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_camera);
if (null == savedInstanceState) {
getFragmentManager()
.beginTransaction()
.replace(R.id.container, Camera2BasicFragment.newInstance())
.commit();
}
}
}
@@ -0,0 +1,373 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package com.example.android.tflitecamerademo;
import android.app.Activity;
import android.content.res.AssetFileDescriptor;
import android.graphics.Bitmap;
import android.os.SystemClock;
import android.text.SpannableString;
import android.text.SpannableStringBuilder;
import android.text.style.ForegroundColorSpan;
import android.text.style.RelativeSizeSpan;
import android.util.Log;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
import org.tensorflow.lite.Interpreter;
import org.tensorflow.lite.gpu.GpuDelegate;
import org.tensorflow.lite.nnapi.NnApiDelegate;
/**
* Classifies images with Tensorflow Lite.
*/
public abstract class ImageClassifier {
// Display preferences
private static final float GOOD_PROB_THRESHOLD = 0.3f;
private static final int SMALL_COLOR = 0xffddaa88;
/** Tag for the {@link Log}. */
private static final String TAG = "TfLiteCameraDemo";
/** Number of results to show in the UI. */
private static final int RESULTS_TO_SHOW = 3;
/** Dimensions of inputs. */
private static final int DIM_BATCH_SIZE = 1;
private static final int DIM_PIXEL_SIZE = 3;
/** Preallocated buffers for storing image data in. */
private int[] intValues = new int[getImageSizeX() * getImageSizeY()];
/** Options for configuring the Interpreter. */
private final Interpreter.Options tfliteOptions = new Interpreter.Options();
/** The loaded TensorFlow Lite model. */
private MappedByteBuffer tfliteModel;
/** An instance of the driver class to run model inference with Tensorflow Lite. */
protected Interpreter tflite;
/** Labels corresponding to the output of the vision model. */
private List<String> labelList;
/** A ByteBuffer to hold image data, to be feed into Tensorflow Lite as inputs. */
protected ByteBuffer imgData = null;
/** multi-stage low pass filter * */
private float[][] filterLabelProbArray = null;
private static final int FILTER_STAGES = 3;
private static final float FILTER_FACTOR = 0.4f;
private PriorityQueue<Map.Entry<String, Float>> sortedLabels =
new PriorityQueue<>(
RESULTS_TO_SHOW,
new Comparator<Map.Entry<String, Float>>() {
@Override
public int compare(Map.Entry<String, Float> o1, Map.Entry<String, Float> o2) {
return (o1.getValue()).compareTo(o2.getValue());
}
});
/** holds a gpu delegate */
GpuDelegate gpuDelegate = null;
/** holds an nnapi delegate */
NnApiDelegate nnapiDelegate = null;
/** Initializes an {@code ImageClassifier}. */
ImageClassifier(Activity activity) throws IOException {
tfliteModel = loadModelFile(activity);
tflite = new Interpreter(tfliteModel, tfliteOptions);
labelList = loadLabelList(activity);
imgData =
ByteBuffer.allocateDirect(
DIM_BATCH_SIZE
* getImageSizeX()
* getImageSizeY()
* DIM_PIXEL_SIZE
* getNumBytesPerChannel());
imgData.order(ByteOrder.nativeOrder());
filterLabelProbArray = new float[FILTER_STAGES][getNumLabels()];
Log.d(TAG, "Created a Tensorflow Lite Image Classifier.");
}
/** Classifies a frame from the preview stream. */
void classifyFrame(Bitmap bitmap, SpannableStringBuilder builder) {
if (tflite == null) {
Log.e(TAG, "Image classifier has not been initialized; Skipped.");
builder.append(new SpannableString("Uninitialized Classifier."));
}
convertBitmapToByteBuffer(bitmap);
// Here's where the magic happens!!!
long startTime = SystemClock.uptimeMillis();
runInference();
long endTime = SystemClock.uptimeMillis();
Log.d(TAG, "Timecost to run model inference: " + Long.toString(endTime - startTime));
// Smooth the results across frames.
applyFilter();
// Print the results.
printTopKLabels(builder);
long duration = endTime - startTime;
SpannableString span = new SpannableString(duration + " ms");
span.setSpan(new ForegroundColorSpan(android.graphics.Color.LTGRAY), 0, span.length(), 0);
builder.append(span);
}
void applyFilter() {
int numLabels = getNumLabels();
// Low pass filter `labelProbArray` into the first stage of the filter.
for (int j = 0; j < numLabels; ++j) {
filterLabelProbArray[0][j] +=
FILTER_FACTOR * (getProbability(j) - filterLabelProbArray[0][j]);
}
// Low pass filter each stage into the next.
for (int i = 1; i < FILTER_STAGES; ++i) {
for (int j = 0; j < numLabels; ++j) {
filterLabelProbArray[i][j] +=
FILTER_FACTOR * (filterLabelProbArray[i - 1][j] - filterLabelProbArray[i][j]);
}
}
// Copy the last stage filter output back to `labelProbArray`.
for (int j = 0; j < numLabels; ++j) {
setProbability(j, filterLabelProbArray[FILTER_STAGES - 1][j]);
}
}
private void recreateInterpreter() {
if (tflite != null) {
tflite.close();
tflite = new Interpreter(tfliteModel, tfliteOptions);
}
}
public void useGpu() {
if (gpuDelegate == null) {
GpuDelegate.Options options = new GpuDelegate.Options();
options.setQuantizedModelsAllowed(true);
gpuDelegate = new GpuDelegate(options);
tfliteOptions.addDelegate(gpuDelegate);
recreateInterpreter();
}
}
public void useCPU() {
recreateInterpreter();
}
public void useNNAPI() {
nnapiDelegate = new NnApiDelegate();
tfliteOptions.addDelegate(nnapiDelegate);
recreateInterpreter();
}
public void setNumThreads(int numThreads) {
tfliteOptions.setNumThreads(numThreads);
recreateInterpreter();
}
/** Closes tflite to release resources. */
public void close() {
tflite.close();
tflite = null;
if (gpuDelegate != null) {
gpuDelegate.close();
gpuDelegate = null;
}
if (nnapiDelegate != null) {
nnapiDelegate.close();
nnapiDelegate = null;
}
tfliteModel = null;
}
/** Reads label list from Assets. */
private List<String> loadLabelList(Activity activity) throws IOException {
List<String> labelList = new ArrayList<String>();
BufferedReader reader =
new BufferedReader(new InputStreamReader(activity.getAssets().open(getLabelPath())));
String line;
while ((line = reader.readLine()) != null) {
labelList.add(line);
}
reader.close();
return labelList;
}
/** Memory-map the model file in Assets. */
private MappedByteBuffer loadModelFile(Activity activity) throws IOException {
AssetFileDescriptor fileDescriptor = activity.getAssets().openFd(getModelPath());
FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor());
FileChannel fileChannel = inputStream.getChannel();
long startOffset = fileDescriptor.getStartOffset();
long declaredLength = fileDescriptor.getDeclaredLength();
return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
}
/** Writes Image data into a {@code ByteBuffer}. */
private void convertBitmapToByteBuffer(Bitmap bitmap) {
if (imgData == null) {
return;
}
imgData.rewind();
bitmap.getPixels(intValues, 0, bitmap.getWidth(), 0, 0, bitmap.getWidth(), bitmap.getHeight());
// Convert the image to floating point.
int pixel = 0;
long startTime = SystemClock.uptimeMillis();
for (int i = 0; i < getImageSizeX(); ++i) {
for (int j = 0; j < getImageSizeY(); ++j) {
final int val = intValues[pixel++];
addPixelValue(val);
}
}
long endTime = SystemClock.uptimeMillis();
Log.d(TAG, "Timecost to put values into ByteBuffer: " + Long.toString(endTime - startTime));
}
/** Prints top-K labels, to be shown in UI as the results. */
private void printTopKLabels(SpannableStringBuilder builder) {
for (int i = 0; i < getNumLabels(); ++i) {
sortedLabels.add(
new AbstractMap.SimpleEntry<>(labelList.get(i), getNormalizedProbability(i)));
if (sortedLabels.size() > RESULTS_TO_SHOW) {
sortedLabels.poll();
}
}
final int size = sortedLabels.size();
for (int i = 0; i < size; i++) {
Map.Entry<String, Float> label = sortedLabels.poll();
SpannableString span =
new SpannableString(String.format("%s: %4.2f\n", label.getKey(), label.getValue()));
int color;
// Make it white when probability larger than threshold.
if (label.getValue() > GOOD_PROB_THRESHOLD) {
color = android.graphics.Color.WHITE;
} else {
color = SMALL_COLOR;
}
// Make first item bigger.
if (i == size - 1) {
float sizeScale = (i == size - 1) ? 1.25f : 0.8f;
span.setSpan(new RelativeSizeSpan(sizeScale), 0, span.length(), 0);
}
span.setSpan(new ForegroundColorSpan(color), 0, span.length(), 0);
builder.insert(0, span);
}
}
/**
* Get the name of the model file stored in Assets.
*
* @return
*/
protected abstract String getModelPath();
/**
* Get the name of the label file stored in Assets.
*
* @return
*/
protected abstract String getLabelPath();
/**
* Get the image size along the x axis.
*
* @return
*/
protected abstract int getImageSizeX();
/**
* Get the image size along the y axis.
*
* @return
*/
protected abstract int getImageSizeY();
/**
* Get the number of bytes that is used to store a single color channel value.
*
* @return
*/
protected abstract int getNumBytesPerChannel();
/**
* Add pixelValue to byteBuffer.
*
* @param pixelValue
*/
protected abstract void addPixelValue(int pixelValue);
/**
* Read the probability value for the specified label This is either the original value as it was
* read from the net's output or the updated value after the filter was applied.
*
* @param labelIndex
* @return
*/
protected abstract float getProbability(int labelIndex);
/**
* Set the probability value for the specified label.
*
* @param labelIndex
* @param value
*/
protected abstract void setProbability(int labelIndex, Number value);
/**
* Get the normalized probability value for the specified label. This is the final value as it
* will be shown to the user.
*
* @return
*/
protected abstract float getNormalizedProbability(int labelIndex);
/**
* Run inference using the prepared input in {@link #imgData}. Afterwards, the result will be
* provided by getProbability().
*
* <p>This additional method is necessary, because we don't have a common base for different
* primitive data types.
*/
protected abstract void runInference();
/**
* Get the total number of labels.
*
* @return
*/
protected int getNumLabels() {
return labelList.size();
}
}
@@ -0,0 +1,105 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package com.example.android.tflitecamerademo;
import android.app.Activity;
import java.io.IOException;
/**
* This classifier works with the Inception-v3 slim model.
* It applies floating point inference rather than using a quantized model.
*/
public class ImageClassifierFloatInception extends ImageClassifier {
/**
* The inception net requires additional normalization of the used input.
*/
private static final int IMAGE_MEAN = 128;
private static final float IMAGE_STD = 128.0f;
/**
* An array to hold inference results, to be feed into Tensorflow Lite as outputs.
* This isn't part of the super class, because we need a primitive array here.
*/
private float[][] labelProbArray = null;
/**
* Initializes an {@code ImageClassifier}.
*
* @param activity
*/
ImageClassifierFloatInception(Activity activity) throws IOException {
super(activity);
labelProbArray = new float[1][getNumLabels()];
}
@Override
protected String getModelPath() {
// you can download this file from
// https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip
return "inceptionv3_slim_2016.tflite";
}
@Override
protected String getLabelPath() {
return "labels_imagenet_slim.txt";
}
@Override
protected int getImageSizeX() {
return 299;
}
@Override
protected int getImageSizeY() {
return 299;
}
@Override
protected int getNumBytesPerChannel() {
// a 32bit float value requires 4 bytes
return 4;
}
@Override
protected void addPixelValue(int pixelValue) {
imgData.putFloat((((pixelValue >> 16) & 0xFF) - IMAGE_MEAN) / IMAGE_STD);
imgData.putFloat((((pixelValue >> 8) & 0xFF) - IMAGE_MEAN) / IMAGE_STD);
imgData.putFloat(((pixelValue & 0xFF) - IMAGE_MEAN) / IMAGE_STD);
}
@Override
protected float getProbability(int labelIndex) {
return labelProbArray[0][labelIndex];
}
@Override
protected void setProbability(int labelIndex, Number value) {
labelProbArray[0][labelIndex] = value.floatValue();
}
@Override
protected float getNormalizedProbability(int labelIndex) {
// TODO the following value isn't in [0,1] yet, but may be greater. Why?
return getProbability(labelIndex);
}
@Override
protected void runInference() {
tflite.run(imgData, labelProbArray);
}
}
@@ -0,0 +1,99 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package com.example.android.tflitecamerademo;
import android.app.Activity;
import java.io.IOException;
/** This classifier works with the float MobileNet model. */
public class ImageClassifierFloatMobileNet extends ImageClassifier {
/** The mobile net requires additional normalization of the used input. */
private static final float IMAGE_MEAN = 127.5f;
private static final float IMAGE_STD = 127.5f;
/**
* An array to hold inference results, to be feed into Tensorflow Lite as outputs. This isn't part
* of the super class, because we need a primitive array here.
*/
private float[][] labelProbArray = null;
/**
* Initializes an {@code ImageClassifierFloatMobileNet}.
*
* @param activity
*/
ImageClassifierFloatMobileNet(Activity activity) throws IOException {
super(activity);
labelProbArray = new float[1][getNumLabels()];
}
@Override
protected String getModelPath() {
// you can download this file from
// see build.gradle for where to obtain this file. It should be auto
// downloaded into assets.
return "mobilenet_v1_1.0_224.tflite";
}
@Override
protected String getLabelPath() {
return "labels_mobilenet_quant_v1_224.txt";
}
@Override
protected int getImageSizeX() {
return 224;
}
@Override
protected int getImageSizeY() {
return 224;
}
@Override
protected int getNumBytesPerChannel() {
return 4; // Float.SIZE / Byte.SIZE;
}
@Override
protected void addPixelValue(int pixelValue) {
imgData.putFloat((((pixelValue >> 16) & 0xFF) - IMAGE_MEAN) / IMAGE_STD);
imgData.putFloat((((pixelValue >> 8) & 0xFF) - IMAGE_MEAN) / IMAGE_STD);
imgData.putFloat(((pixelValue & 0xFF) - IMAGE_MEAN) / IMAGE_STD);
}
@Override
protected float getProbability(int labelIndex) {
return labelProbArray[0][labelIndex];
}
@Override
protected void setProbability(int labelIndex, Number value) {
labelProbArray[0][labelIndex] = value.floatValue();
}
@Override
protected float getNormalizedProbability(int labelIndex) {
return labelProbArray[0][labelIndex];
}
@Override
protected void runInference() {
tflite.run(imgData, labelProbArray);
}
}
@@ -0,0 +1,97 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package com.example.android.tflitecamerademo;
import android.app.Activity;
import java.io.IOException;
/**
* This classifier works with the quantized MobileNet model.
*/
public class ImageClassifierQuantizedMobileNet extends ImageClassifier {
/**
* An array to hold inference results, to be feed into Tensorflow Lite as outputs.
* This isn't part of the super class, because we need a primitive array here.
*/
private byte[][] labelProbArray = null;
/**
* Initializes an {@code ImageClassifier}.
*
* @param activity
*/
ImageClassifierQuantizedMobileNet(Activity activity) throws IOException {
super(activity);
labelProbArray = new byte[1][getNumLabels()];
}
@Override
protected String getModelPath() {
// you can download this file from
// see build.gradle for where to obtain this file. It should be auto
// downloaded into assets.
return "mobilenet_v1_1.0_224_quant.tflite";
}
@Override
protected String getLabelPath() {
return "labels_mobilenet_quant_v1_224.txt";
}
@Override
protected int getImageSizeX() {
return 224;
}
@Override
protected int getImageSizeY() {
return 224;
}
@Override
protected int getNumBytesPerChannel() {
// the quantized model uses a single byte only
return 1;
}
@Override
protected void addPixelValue(int pixelValue) {
imgData.put((byte) ((pixelValue >> 16) & 0xFF));
imgData.put((byte) ((pixelValue >> 8) & 0xFF));
imgData.put((byte) (pixelValue & 0xFF));
}
@Override
protected float getProbability(int labelIndex) {
return labelProbArray[0][labelIndex];
}
@Override
protected void setProbability(int labelIndex, Number value) {
labelProbArray[0][labelIndex] = value.byteValue();
}
@Override
protected float getNormalizedProbability(int labelIndex) {
return (labelProbArray[0][labelIndex] & 0xff) / 255.0f;
}
@Override
protected void runInference() {
tflite.run(imgData, labelProbArray);
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 490 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 116 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 320 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 611 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 952 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<selector xmlns:android="http://schemas.android.com/apk/res/android">
<!-- pressed -->
<item android:drawable="@color/selection_highlight" android:state_pressed="true" />
<!-- focused -->
<item android:drawable="@color/selection_focus" android:state_activated="true" />
<!-- default -->
<item android:drawable="@color/item_normal" />
</selector>
@@ -0,0 +1,118 @@
<?xml version="1.0" encoding="utf-8"?><!--
Copyright 2014 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<LinearLayout
xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools" android:layout_width="match_parent"
android:layout_height="match_parent"
android:background="#bb7700"
android:orientation="horizontal">
<com.example.android.tflitecamerademo.AutoFitTextureView
android:id="@+id/texture"
android:layout_width="0dp"
android:layout_height="match_parent"
android:layout_weight=".8"/>
<LinearLayout
android:layout_width="0dp"
android:layout_height="match_parent"
android:layout_weight=".2"
android:orientation="vertical">
<ImageView
android:id="@+id/logoview"
android:layout_width="wrap_content"
android:layout_height="47dp"
android:scaleType="centerInside"
android:src="@drawable/logo"/>
<TextView
android:id="@+id/text"
android:layout_width="match_parent"
android:layout_height="160dp"
android:paddingTop="20dp"
android:textColor="#FFF"
android:textSize="20sp"
android:textStyle="bold"/>
<LinearLayout
android:id="@+id/modelLayout"
android:layout_width="match_parent"
android:layout_height="150dp"
android:orientation="vertical">
<TextView
android:id="@+id/textView"
android:layout_width="match_parent"
android:layout_height="20dp"
android:text="@string/modelLabel"
android:textAlignment="center"
android:textColor="@android:color/white"/>
<ListView
android:id="@+id/model"
android:layout_width="match_parent"
android:layout_height="wrap_content">
</ListView>
</LinearLayout>
<LinearLayout
android:id="@+id/deviceLayout"
android:layout_width="match_parent"
android:layout_height="150dp"
android:orientation="vertical">
<TextView
android:id="@+id/textView2"
android:layout_width="match_parent"
android:layout_height="20dp"
android:text="@string/deviceLabel"
android:textAlignment="center"
android:textColor="@android:color/white"/>
<ListView
android:id="@+id/device"
android:layout_width="match_parent"
android:layout_height="wrap_content"/>
</LinearLayout>
<LinearLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
>
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:gravity="center"
android:text="Threads"
android:textAlignment="center"
android:textColor="@android:color/white"/>
<NumberPicker
android:id="@+id/np"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginLeft="10dp"
android:theme="@style/AppTheme.Picker"
android:visibility="visible"/>
</LinearLayout>
</LinearLayout>
</LinearLayout>
@@ -0,0 +1,140 @@
<?xml version="1.0" encoding="utf-8"?><!--
Copyright 2014 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:background="#bb7700">
<com.example.android.tflitecamerademo.AutoFitTextureView
android:id="@+id/texture"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:layout_weight="1" />
<LinearLayout
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_above="@+id/bottom_info_view"
android:layout_alignParentEnd="false"
android:layout_alignParentStart="true"
android:layout_alignParentTop="false"
android:background="#bb7700"
android:orientation="vertical"
android:weightSum="100">
<ImageView
android:id="@+id/logoview2"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_weight="30"
android:scaleType="fitStart"
android:src="@drawable/logo" />
<TextView
android:id="@+id/text"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_alignParentBottom="true"
android:layout_alignParentEnd="true"
android:layout_alignParentRight="true"
android:layout_weight="30"
android:textColor="#FFF"
android:textSize="20sp"
android:textStyle="bold" />
</LinearLayout>
<LinearLayout
android:id="@+id/bottom_info_view"
android:layout_width="match_parent"
android:layout_height="200dp"
android:layout_alignParentBottom="true"
android:layout_marginBottom="10dp"
android:background="#513400"
android:orientation="horizontal">
<LinearLayout
android:layout_width="wrap_content"
android:layout_height="match_parent"
android:orientation="vertical">
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:gravity="center"
android:text="Threads"
android:textAlignment="center"
android:textColor="@android:color/white" />
<NumberPicker
android:id="@+id/np"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginLeft="10dp"
android:theme="@style/AppTheme.Picker"
android:visibility="visible" />
</LinearLayout>
<LinearLayout
android:id="@+id/modelLayout"
android:layout_width="150dp"
android:layout_height="match_parent"
android:orientation="vertical">
<TextView
android:id="@+id/textView"
android:layout_width="match_parent"
android:layout_height="20dp"
android:text="@string/modelLabel"
android:textAlignment="center"
android:textColor="@android:color/white" />
<ListView
android:id="@+id/model"
android:layout_width="match_parent"
android:layout_height="180dp">
</ListView>
</LinearLayout>
<LinearLayout
android:id="@+id/deviceLayout"
android:layout_width="140dp"
android:layout_height="match_parent"
android:orientation="vertical">
<TextView
android:id="@+id/textView2"
android:layout_width="match_parent"
android:layout_height="20dp"
android:text="@string/deviceLabel"
android:textAlignment="center"
android:textColor="@android:color/white" />
<ListView
android:id="@+id/device"
android:layout_width="match_parent"
android:layout_height="180dp" />
</LinearLayout>
</LinearLayout>
</RelativeLayout>
@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="utf-8"?><!--
Copyright 2014 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools"
android:id="@+id/container"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:background="#000"
tools:context="com.example.android.tflitecamerademo.CameraActivity" />
@@ -0,0 +1,138 @@
<?xml version="1.0" encoding="utf-8"?><!--
Copyright 2014 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:background="#bb7700">
<com.example.android.tflitecamerademo.AutoFitTextureView
android:id="@+id/texture"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:layout_weight="1" />
<LinearLayout
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_above="@+id/bottom_info_view"
android:layout_alignParentEnd="false"
android:layout_alignParentStart="true"
android:layout_alignParentTop="false"
android:background="#bb7700"
android:orientation="vertical"
android:weightSum="100">
<ImageView
android:id="@+id/logoview2"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_weight="30"
android:scaleType="fitStart"
android:src="@drawable/logo" />
<TextView
android:id="@+id/text"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_alignParentBottom="true"
android:layout_alignParentEnd="true"
android:layout_alignParentRight="true"
android:layout_weight="30"
android:textColor="#FFF"
android:textSize="20sp"
android:textStyle="bold" />
</LinearLayout>
<LinearLayout
android:id="@+id/bottom_info_view"
android:layout_width="match_parent"
android:layout_height="200dp"
android:layout_alignParentBottom="true"
android:layout_marginBottom="10dp"
android:background="#513400"
android:orientation="horizontal">
<LinearLayout
android:layout_width="wrap_content"
android:layout_height="match_parent"
android:orientation="vertical">
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:gravity="center"
android:text="Threads"
android:textAlignment="center"
android:textColor="@android:color/white" />
<NumberPicker
android:id="@+id/np"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginLeft="10dp"
android:theme="@style/AppTheme.Picker"
android:visibility="visible" />
</LinearLayout>
<LinearLayout
android:id="@+id/modelLayout"
android:layout_width="150dp"
android:layout_height="match_parent"
android:orientation="vertical">
<TextView
android:id="@+id/textView"
android:layout_width="match_parent"
android:layout_height="20dp"
android:text="@string/modelLabel"
android:textAlignment="center"
android:textColor="@android:color/white" />
<ListView
android:id="@+id/model"
android:layout_width="match_parent"
android:layout_height="180dp">
</ListView>
</LinearLayout>
<LinearLayout
android:id="@+id/deviceLayout"
android:layout_width="140dp"
android:layout_height="match_parent"
android:orientation="vertical">
<TextView
android:id="@+id/textView2"
android:layout_width="match_parent"
android:layout_height="20dp"
android:text="@string/deviceLabel"
android:textAlignment="center"
android:textColor="@android:color/white" />
<ListView
android:id="@+id/device"
android:layout_width="match_parent"
android:layout_height="180dp" />
</LinearLayout>
</LinearLayout>
</RelativeLayout>
@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:layout_width="match_parent"
android:layout_height="match_parent">
<TextView
android:id="@+id/listview_row_text"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:layout_marginRight="2dp"
android:background="@drawable/item_selector"
android:padding="10dp"
android:textSize="18sp"
android:textStyle="bold" />
</LinearLayout>
@@ -0,0 +1,24 @@
<!--
Copyright 2013 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources>
<!-- Semantic definitions -->
<dimen name="horizontal_page_margin">@dimen/margin_huge</dimen>
<dimen name="vertical_page_margin">@dimen/margin_medium</dimen>
</resources>
@@ -0,0 +1,25 @@
<!--
Copyright 2013 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources>
<style name="Widget.SampleMessage">
<item name="android:textAppearance">?android:textAppearanceLarge</item>
<item name="android:lineSpacingMultiplier">1.2</item>
<item name="android:shadowDy">-6.5</item>
</style>
</resources>
@@ -0,0 +1,22 @@
<!--
Copyright 2013 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources>
<!-- Activity themes -->
<style name="Theme.Base" parent="android:Theme.Holo.Light" />
</resources>
@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright 2013 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources>
</resources>
@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright 2013 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources>
<!-- Activity themes -->
<style name="Theme.Base" parent="android:Theme.Material.Light">
</style>
</resources>
@@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright 2013 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources>
<string name="app_name">TfLite Camera Demo</string>
<string name="intro_message">
<![CDATA[
This sample demonstrates the basic use of TfLite API. Check the source code to see how
you can use TfLite for efficient, on-device inference with trained TensorFlow models.
]]>
</string>
<string name="threads">Threads:</string>
</resources>
@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
Copyright 2015 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources>
<color name="control_background">#cc4285f4</color>
<color name="selection_highlight">#aaaaaa</color>
<color name="selection_focus">#eeaa55</color>
<color name="item_normal">#eeeeee</color>
</resources>
@@ -0,0 +1,33 @@
<?xml version="1.0" encoding="utf-8"?><!--
Copyright 2014 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources>
<string name="picture">Picture</string>
<string name="description_info">Info</string>
<string name="request_permission">This sample needs camera permission.</string>
<string name="camera_error">This device doesn\'t support Camera2 API.</string>
<string name="toggle_turn_on">NN:On</string>
<string name="toggle_turn_off">NN:Off</string>
<string name="toggle">Use NNAPI</string>
<string name="tflite">tflite</string>
<string name="nnapi">NNAPI</string>
<string name="gpu">GPU</string>
<string name="cpu">CPU</string>
<string name="modelLabel">Model</string>
<string name="deviceLabel">Device</string>
<string name="mobilenetV1Quant">mobilenet v1 quant</string>
<string name="mobilenetV1Float">mobilenet v1 float</string>
</resources>
@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="utf-8"?><!--
Copyright 2014 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources>
<style name="MaterialTheme" parent="android:Theme.Material.Light.NoActionBar.Fullscreen" />
<style name="AppTheme.Picker" parent="android:Theme.Material.Light.NoActionBar.Fullscreen" >
<item name="android:textColorPrimary">@android:color/white</item>
</style>
</resources>
@@ -0,0 +1,32 @@
<!--
Copyright 2013 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources>
<!-- Define standard dimensions to comply with Holo-style grids and rhythm. -->
<dimen name="margin_tiny">4dp</dimen>
<dimen name="margin_small">8dp</dimen>
<dimen name="margin_medium">16dp</dimen>
<dimen name="margin_large">32dp</dimen>
<dimen name="margin_huge">64dp</dimen>
<!-- Semantic definitions -->
<dimen name="horizontal_page_margin">@dimen/margin_medium</dimen>
<dimen name="vertical_page_margin">@dimen/margin_medium</dimen>
</resources>
@@ -0,0 +1,42 @@
<!--
Copyright 2013 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources>
<!-- Activity themes -->
<style name="Theme.Base" parent="android:Theme.Light" />
<style name="Theme.Sample" parent="Theme.Base" />
<style name="AppTheme" parent="Theme.Sample" />
<!-- Widget styling -->
<style name="Widget" />
<style name="Widget.SampleMessage">
<item name="android:textAppearance">?android:textAppearanceMedium</item>
<item name="android:lineSpacingMultiplier">1.1</item>
</style>
<style name="Widget.SampleMessageTile">
<item name="android:background">@drawable/tile</item>
<item name="android:shadowColor">#7F000000</item>
<item name="android:shadowDy">-3.5</item>
<item name="android:shadowRadius">2</item>
</style>
</resources>
+29
View File
@@ -0,0 +1,29 @@
// Top-level build file where you can add configuration options common to all sub-projects/modules.
buildscript {
repositories {
google()
mavenCentral()
}
dependencies {
classpath 'com.android.tools.build:gradle:3.1.4'
// NOTE: Do not place your application dependencies here; they belong
// in the individual module build.gradle files
}
}
allprojects {
repositories {
google()
mavenCentral()
maven { // Only for snapshot artifacts
name 'ossrh-snapshot'
url 'https://oss.sonatype.org/content/repositories/snapshots'
}
}
}
task clean(type: Delete) {
delete rootProject.buildDir
}
@@ -0,0 +1,17 @@
# Project-wide Gradle settings.
# IDE (e.g. Android Studio) users:
# Gradle settings configured through the IDE *will override*
# any settings specified in this file.
# For more details on how to configure your build environment visit
# http://www.gradle.org/docs/current/userguide/build_environment.html
# Specifies the JVM arguments used for the daemon process.
# The setting is particularly useful for tweaking memory settings.
org.gradle.jvmargs=-Xmx1536m
# When configured, Gradle will run in incubating parallel mode.
# This option should only be used with decoupled projects. More details, visit
# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
# org.gradle.parallel=true
Binary file not shown.
@@ -0,0 +1,6 @@
#Thu Sep 28 09:01:41 PDT 2017
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-all.zip
+160
View File
@@ -0,0 +1,160 @@
#!/usr/bin/env bash
##############################################################################
##
## Gradle start up script for UN*X
##
##############################################################################
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS=""
APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
warn ( ) {
echo "$*"
}
die ( ) {
echo
echo "$*"
echo
exit 1
}
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
case "`uname`" in
CYGWIN* )
cygwin=true
;;
Darwin* )
darwin=true
;;
MINGW* )
msys=true
;;
esac
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
PRG="$0"
# Need this for relative symlinks.
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "$PRG"`"/$link"
fi
done
SAVED="`pwd`"
cd "`dirname \"$PRG\"`/" >/dev/null
APP_HOME="`pwd -P`"
cd "$SAVED" >/dev/null
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD="java"
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
MAX_FD_LIMIT=`ulimit -H -n`
if [ $? -eq 0 ] ; then
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
MAX_FD="$MAX_FD_LIMIT"
fi
ulimit -n $MAX_FD
if [ $? -ne 0 ] ; then
warn "Could not set maximum file descriptor limit: $MAX_FD"
fi
else
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
fi
fi
# For Darwin, add options to specify how the application appears in the dock
if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi
# For Cygwin, switch paths to Windows format before running java
if $cygwin ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
JAVACMD=`cygpath --unix "$JAVACMD"`
# We build the pattern for arguments to be converted via cygpath
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
SEP=""
for dir in $ROOTDIRSRAW ; do
ROOTDIRS="$ROOTDIRS$SEP$dir"
SEP="|"
done
OURCYGPATTERN="(^($ROOTDIRS))"
# Add a user-defined pattern to the cygpath arguments
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
fi
# Now convert the arguments - kludge to limit ourselves to /bin/sh
i=0
for arg in "$@" ; do
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
else
eval `echo args$i`="\"$arg\""
fi
i=$((i+1))
done
case $i in
(0) set -- ;;
(1) set -- "$args0" ;;
(2) set -- "$args0" "$args1" ;;
(3) set -- "$args0" "$args1" "$args2" ;;
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi
# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
function splitJvmOpts() {
JVM_OPTS=("$@")
}
eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+105
View File
@@ -0,0 +1,105 @@
:: Copyright 2026 The TensorFlow Authors. All Rights Reserved.
::
:: Licensed under the Apache License, Version 2.0 (the "License");
:: you may not use this file except in compliance with the License.
:: You may obtain a copy of the License at
::
:: http://www.apache.org/licenses/LICENSE-2.0
::
:: Unless required by applicable law or agreed to in writing, software
:: distributed under the License is distributed on an "AS IS" BASIS,
:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
:: See the License for the specific language governing permissions and
:: limitations under the License.
:: ==============================================================================
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@rem Gradle startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables with windows NT shell
if "%OS%"=="Windows_NT" setlocal
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS=
set DIRNAME=%~dp0
if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if "%ERRORLEVEL%" == "0" goto init
echo.
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto init
echo.
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:init
@rem Get command-line arguments, handling Windowz variants
if not "%OS%" == "Windows_NT" goto win9xME_args
if "%@eval[2+2]" == "4" goto 4NT_args
:win9xME_args
@rem Slurp the command line arguments.
set CMD_LINE_ARGS=
set _SKIP=2
:win9xME_args_slurp
if "x%~1" == "x" goto execute
set CMD_LINE_ARGS=%*
goto execute
:4NT_args
@rem Get arguments from the 4NT Shell from JP Software
set CMD_LINE_ARGS=%$
:execute
@rem Setup the command line
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
:end
@rem End local scope for the variables with windows NT shell
if "%ERRORLEVEL%"=="0" goto mainEnd
:fail
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
rem the _cmd.exe /c_ return code!
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
exit /b 1
:mainEnd
if "%OS%"=="Windows_NT" endlocal
:omega
@@ -0,0 +1 @@
include ':app'
@@ -0,0 +1,12 @@
VERS_1.0 {
# Export JNI and native C symbols.
global:
Java_*;
JNI_OnLoad;
JNI_OnUnload;
TfLiteGpu*;
# Hide everything else.
local:
*;
};
+33
View File
@@ -0,0 +1,33 @@
load("@rules_cc//cc:cc_library.bzl", "cc_library")
load("//tensorflow:tensorflow.default.bzl", "get_compatible_with_portable")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
licenses = ["notice"],
)
# We need special handling for JNI inclusion for Android. Rather than duplicating this logic
# for every target that uses JNI, we use a single proxy target that
# encapsulates it.
alias(
name = "jni",
actual = select({
# The Android toolchain makes <jni.h> available in the system include
# path.
# Aliases need to resolve to a single target however, so alias to an
# empty library instead.
# (Making this target a cc_library with empty deps for the Android case
# doesn't work, because go/cpp-features#layering-check requires targets
# to _directly_ depend on libraries they include, and cc_library doesn't
# have any direct equivalent to java_library's 'export' attribute).
"//tensorflow:android": ":empty",
# For non-Android toolchains, depend on the JDK JNI headers.
"//conditions:default": "@bazel_tools//tools/jdk:jni",
}),
visibility = ["//visibility:public"],
)
cc_library(
name = "empty",
compatible_with = get_compatible_with_portable(),
)
+132
View File
@@ -0,0 +1,132 @@
# Description:
# OVIC Benchmarker Java API.
load("@build_bazel_rules_android//android:rules.bzl", "android_library")
load("@rules_java//java:defs.bzl", "java_binary", "java_library", "java_test")
load("//tensorflow/java:build_defs.bzl", "JAVACOPTS")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
default_visibility = ["//visibility:public"],
licenses = ["notice"],
)
# Build targets for OVIC classification.
java_test(
name = "OvicClassifierTest",
size = "medium",
srcs = ["src/test/java/org/tensorflow/ovic/OvicClassifierTest.java"],
data = [
"//tensorflow/lite/java/ovic/src/testdata:labels.txt",
"//tensorflow/lite/java/ovic/src/testdata:ovic_testdata",
],
javacopts = JAVACOPTS,
tags = [
"no_oss",
],
test_class = "org.tensorflow.ovic.OvicClassifierTest",
visibility = ["//visibility:public"],
deps = [
"//tensorflow/lite/java/ovic:ovicbenchmarkerlib_java",
"@com_google_truth",
"@junit",
],
)
java_binary(
name = "ovic_validator",
srcs = ["src/main/java/org/tensorflow/ovic/OvicValidator.java"],
data = [
"//tensorflow/lite/java/ovic/src/testdata:labels.txt",
],
main_class = "org.tensorflow.ovic.OvicValidator",
tags = ["no_oss"],
deps = [
"//tensorflow/lite/java/ovic:ovicbenchmarkerlib_java",
"//tensorflow/lite/java/ovic:ovicdetectionbenchmarkerlib_java",
],
)
android_library(
name = "ovicbenchmarkerlib",
srcs = [
"src/main/java/org/tensorflow/ovic/OvicBenchmarker.java",
"src/main/java/org/tensorflow/ovic/OvicClassificationResult.java",
"src/main/java/org/tensorflow/ovic/OvicClassifier.java",
"src/main/java/org/tensorflow/ovic/OvicClassifierBenchmarker.java",
],
tags = ["no_oss"],
deps = [
"//tensorflow/lite/java:tensorflowlite",
"//tensorflow/lite/java/src/testhelper/java/org/tensorflow/lite:testhelper",
],
)
java_library(
name = "ovicbenchmarkerlib_java",
srcs = [
"src/main/java/org/tensorflow/ovic/OvicClassificationResult.java",
"src/main/java/org/tensorflow/ovic/OvicClassifier.java",
],
javacopts = JAVACOPTS,
tags = ["no_oss"],
deps = [
"//tensorflow/lite/java:libtensorflowlite_jni.so",
"//tensorflow/lite/java:tensorflowlite_javalib",
"//tensorflow/lite/java/src/main/native",
"//tensorflow/lite/java/src/testhelper/java/org/tensorflow/lite:testhelper_javalib",
],
)
# Build targets for OVIC detection.
java_test(
name = "OvicDetectorTest",
size = "medium",
srcs = ["src/test/java/org/tensorflow/ovic/OvicDetectorTest.java"],
data = [
"//tensorflow/lite/java/ovic/src/testdata:coco_labels.txt",
"//tensorflow/lite/java/ovic/src/testdata:ovic_testdata",
],
javacopts = JAVACOPTS,
tags = [
"no_oss",
],
test_class = "org.tensorflow.ovic.OvicDetectorTest",
visibility = ["//visibility:public"],
deps = [
"//tensorflow/lite/java/ovic:ovicdetectionbenchmarkerlib_java",
"@com_google_truth",
"@junit",
],
)
android_library(
name = "ovicdetectionbenchmarkerlib",
srcs = [
"src/main/java/org/tensorflow/ovic/BoundingBox.java",
"src/main/java/org/tensorflow/ovic/OvicBenchmarker.java",
"src/main/java/org/tensorflow/ovic/OvicDetectionResult.java",
"src/main/java/org/tensorflow/ovic/OvicDetector.java",
"src/main/java/org/tensorflow/ovic/OvicDetectorBenchmarker.java",
],
deps = [
"//tensorflow/lite/java:tensorflowlite",
"//tensorflow/lite/java/src/testhelper/java/org/tensorflow/lite:testhelper",
],
)
java_library(
name = "ovicdetectionbenchmarkerlib_java",
srcs = [
"src/main/java/org/tensorflow/ovic/BoundingBox.java",
"src/main/java/org/tensorflow/ovic/OvicDetectionResult.java",
"src/main/java/org/tensorflow/ovic/OvicDetector.java",
],
javacopts = JAVACOPTS,
deps = [
"//tensorflow/lite/java:libtensorflowlite_jni.so",
"//tensorflow/lite/java:tensorflowlite_javalib",
"//tensorflow/lite/java/src/main/native",
"//tensorflow/lite/java/src/testhelper/java/org/tensorflow/lite:testhelper_javalib",
],
)
+321
View File
@@ -0,0 +1,321 @@
# OVIC Benchmarker for LPCV 2020
This folder contains the SDK for track one of the
[Low Power Computer Vision workshop at CVPR 2020.](https://lpcv.ai/2020CVPR/ovic-track)
## Pre-requisite
Follow the steps [here](https://www.tensorflow.org/lite/demo_android) to install
Tensorflow, Bazel, and the Android NDK and SDK.
## Test the benchmarker:
The testing utilities helps the developers (you) to make sure that your
submissions in TfLite format will be processed as expected in the competition's
benchmarking system.
Note: for now the tests only provides correctness checks, i.e. classifier
predicts the correct category on the test image, but no on-device latency
measurements. To test the latency measurement functionality, the tests will
print the latency running on a desktop computer, which is not indicative of the
on-device run-time. We are releasing an benchmarker Apk that would allow
developers to measure latency on their own devices.
### Obtain the sample models
The test data (models and images) should be downloaded automatically for you by
Bazel. In case they are not, you can manually install them as below.
Note: all commands should be called from your tensorflow installation folder
(under this folder you should find `tensorflow/lite`).
* Download the
[testdata package](https://storage.googleapis.com/download.tensorflow.org/data/ovic_2019_04_30.zip):
```sh
curl -L https://storage.googleapis.com/download.tensorflow.org/data/ovic_2019_04_30.zip -o /tmp/ovic.zip
```
* Unzip the package into the testdata folder:
```sh
unzip -j /tmp/ovic.zip -d tensorflow/lite/java/ovic/src/testdata/
```
### Run tests
You can run test with Bazel as below. This helps to ensure that the installation
is correct.
```sh
bazel test //tensorflow/lite/java/ovic:OvicClassifierTest --cxxopt=-Wno-all --test_output=all
bazel test //tensorflow/lite/java/ovic:OvicDetectorTest --cxxopt=-Wno-all --test_output=all
```
### Test your submissions
Once you have a submission that follows the instructions from the
[competition site](https://lpcv.ai/2020CVPR/ovic-track), you can verify it in
two ways:
#### Validate using randomly generated images
You can call the validator binary below to verify that your model fits the
format requirements. This often helps you to catch size mismatches (e.g. output
for classification should be [1, 1001] instead of [1,1,1,1001]). Let say the
submission file is located at `/path/to/my_model.lite`, then call:
```sh
bazel build //tensorflow/lite/java/ovic:ovic_validator --cxxopt=-Wno-all
bazel-bin/tensorflow/lite/java/ovic/ovic_validator /path/to/my_model.lite classify
```
Successful validation should print the following message to terminal:
```
Successfully validated /path/to/my_model.lite.
```
To validate detection models, use the same command but provide "detect" as the
second argument instead of "classify".
#### Test that the model produces sensible outcomes
You can go a step further to verify that the model produces results as expected.
This helps you catch bugs during TFLite conversion (e.g. using the wrong mean
and std values).
* Move your submission to the testdata folder:
```sh
cp /path/to/my_model.lite tensorflow/lite/java/ovic/src/testdata/
```
* Resize the test image to the resolutions that are expected by your
submission:
The test images can be found at
`tensorflow/lite/java/ovic/src/testdata/test_image_*.jpg`. You may reuse these
images if your image resolutions are 128x128 or 224x224.
* Add your model and test image to the BUILD rule at
`tensorflow/lite/java/ovic/src/testdata/BUILD`:
```JSON
filegroup(
name = "ovic_testdata",
srcs = [
"@tflite_ovic_testdata//:detect.lite",
"@tflite_ovic_testdata//:float_model.lite",
"@tflite_ovic_testdata//:low_res_model.lite",
"@tflite_ovic_testdata//:quantized_model.lite",
"@tflite_ovic_testdata//:test_image_128.jpg",
"@tflite_ovic_testdata//:test_image_224.jpg"
"my_model.lite", # <--- Your submission.
"my_test_image.jpg", # <--- Your test image.
],
...
```
* For classification models, modify `OvicClassifierTest.java`:
* change `TEST_IMAGE_PATH` to `my_test_image.jpg`.
* change either `FLOAT_MODEL_PATH` or `QUANTIZED_MODEL_PATH` to
`my_model.lite` depending on whether your model runs inference in float
or
[8-bit](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/quantize).
* change `TEST_IMAGE_GROUNDTRUTH` (ImageNet class ID) to be consistent
with your test image.
* For detection models, modify `OvicDetectorTest.java`:
* change `TEST_IMAGE_PATH` to `my_test_image.jpg`.
* change `MODEL_PATH` to `my_model.lite`.
* change `GROUNDTRUTH` (COCO class ID) to be consistent with your test
image.
Now you can run the bazel tests to catch any runtime issues with the submission.
Note: Please make sure that your submission passes the test. If a submission
fails to pass the test it will not be processed by the submission server.
## Measure on-device latency
We provide two ways to measure the on-device latency of your submission. The
first is through our competition server, which is reliable and repeatable, but
is limited to a few trials per day. The second is through the benchmarker Apk,
which requires a device and may not be as accurate as the server, but has a fast
turn-around and no access limitations. We recommend that the participants use
the benchmarker apk for early development, and reserve the competition server
for evaluating promising submissions.
### Running the benchmarker app
Make sure that you have followed instructions in
[Test your submissions](#test-your-submissions) to add your model to the
testdata folder and to the corresponding build rules.
Modify `tensorflow/lite/java/ovic/demo/app/OvicBenchmarkerActivity.java`:
* Add your model to the benchmarker apk by changing `modelPath` and
`testImagePath` to your submission and test image.
```
if (benchmarkClassification) {
...
testImagePath = "my_test_image.jpg";
modelPath = "my_model.lite";
} else { // Benchmarking detection.
...
```
If you are adding a detection model, simply modify `modelPath` and
`testImagePath` in the else block above.
* Adjust the benchmark parameters when needed:
You can change the length of each experiment, and the processor affinity below.
`BIG_CORE_MASK` is an integer whose binary encoding represents the set of used
cores. This number is phone-specific. For example, Pixel 4 has 8 cores: the 4
little cores are represented by the 4 less significant bits, and the 4 big cores
by the 4 more significant bits. Therefore a mask value of 16, or in binary
`00010000`, represents using only the first big core. The mask 32, or in binary
`00100000` uses the second big core and should deliver identical results as the
mask 16 because the big cores are interchangeable.
```
/** Wall time for each benchmarking experiment. */
private static final double WALL_TIME = 3000;
/** Maximum number of iterations in each benchmarking experiment. */
private static final int MAX_ITERATIONS = 100;
/** Mask for binding to a single big core. Pixel 1 (4), Pixel 4 (16). */
private static final int BIG_CORE_MASK = 16;
```
Note: You'll need ROOT access to the phone to change processor affinity.
* Build and install the app.
```
bazel build -c opt --cxxopt=-Wno-all //tensorflow/lite/java/ovic/demo/app:ovic_benchmarker_binary
adb install -r bazel-bin/tensorflow/lite/java/ovic/demo/app/ovic_benchmarker_binary.apk
```
Start the app and pick a task by clicking either the `CLF` button for
classification or the `DET` button for detection. The button should turn bright
green, signaling that the experiment is running. The benchmarking results will
be displayed after about the `WALL_TIME` you specified above. For example:
```
my_model.lite: Average latency=158.6ms after 20 runs.
```
### Sample latencies
Note: the benchmarking results can be quite different depending on the
background processes running on the phone. A few things that help stabilize the
app's readings are placing the phone on a cooling plate, restarting the phone,
and shutting down internet access.
Classification Model | Pixel 1 | Pixel 2 | Pixel 4
-------------------- | :-----: | ------: | :-----:
float_model.lite | 97 | 113 | 37
quantized_model.lite | 73 | 61 | 13
low_res_model.lite | 3 | 3 | 1
Detection Model | Pixel 2 | Pixel 4
---------------------- | :-----: | :-----:
detect.lite | 248 | 82
quantized_detect.lite | 59 | 17
quantized_fpnlite.lite | 96 | 29
All latency numbers are in milliseconds. The Pixel 1 and Pixel 2 latency numbers
are measured on `Oct 17 2019` (Github commit hash
[I05def66f58fa8f2161522f318e00c1b520cf0606](https://github.com/tensorflow/tensorflow/commit/4b02bc0e0ff7a0bc02264bc87528253291b7c949#diff-4e94df4d2961961ba5f69bbd666e0552))
The Pixel 4 latency numbers are measured on `Apr 14 2020` (Github commit hash
[4b2cb67756009dda843c6b56a8b320c8a54373e0](https://github.com/tensorflow/tensorflow/commit/4b2cb67756009dda843c6b56a8b320c8a54373e0)).
Since Pixel 4 has excellent support for 8-bit quantized models, we strongly
recommend you to check out the
[Post-Training Quantization tutorial](https://www.tensorflow.org/lite/performance/post_training_quantization).
The detection models above are both single-shot models (i.e. no object proposal
generation) using TfLite's *fast* version of Non-Max-Suppression (NMS). The fast
NMS is significant faster than the regular NMS (used by the ObjectDetectionAPI
in training) at the expense of about 1% mAP for the listed models.
### Latency table
We have compiled a latency table for common neural network operators such as
convolutions, separable convolutions, and matrix multiplications. The table of
results is available here:
* https://storage.cloud.google.com/ovic-data/
The results were generated by creating a small network containing a single
operation, and running the op under the test harness. For more details see the
NetAdapt paper<sup>1</sup>. We plan to expand table regularly as we test with
newer OS releases and updates to Tensorflow Lite.
### Sample benchmarks
Below are the baseline models (MobileNetV2, MnasNet, and MobileNetV3) used to
compute the reference accuracy for ImageNet classification. The naming
convention of the models are `[precision]_[model
class]_[resolution]_[multiplier]`. Pixel 2 Latency (ms) is measured on a single
Pixel 2 big core using the competition server on `Oct 17 2019`, while Pixel 4
latency (ms) is measured on a single Pixel 4 big core using the competition
server on `Apr 14 2020`. You can find these models on TFLite's
[hosted model page](https://www.tensorflow.org/lite/guide/hosted_models#image_classification).
Model | Pixel 2 | Pixel 4 | Top-1 Accuracy
:---------------------------------: | :-----: | :-----: | :------------:
quant_mobilenetv2_96_35 | 4 | 1 | 0.420
quant_mobilenetv2_96_50 | 5 | 1 | 0.478
quant_mobilenetv2_128_35 | 6 | 2 | 0.474
quant_mobilenetv2_128_50 | 8 | 2 | 0.546
quant_mobilenetv2_160_35 | 9 | 2 | 0.534
quant_mobilenetv2_96_75 | 8 | 2 | 0.560
quant_mobilenetv2_96_100 | 10 | 3 | 0.579
quant_mobilenetv2_160_50 | 12 | 3 | 0.583
quant_mobilenetv2_192_35 | 12 | 3 | 0.557
quant_mobilenetv2_128_75 | 13 | 3 | 0.611
quant_mobilenetv2_192_50 | 16 | 4 | 0.616
quant_mobilenetv2_128_100 | 16 | 4 | 0.629
quant_mobilenetv2_224_35 | 17 | 5 | 0.581
quant_mobilenetv2_160_75 | 20 | 5 | 0.646
float_mnasnet_96_100 | 21 | 7 | 0.625
quant_mobilenetv2_224_50 | 22 | 6 | 0.637
quant_mobilenetv2_160_100 | 25 | 6 | 0.674
quant_mobilenetv2_192_75 | 29 | 7 | 0.674
quant_mobilenetv2_192_100 | 35 | 9 | 0.695
float_mnasnet_224_50 | 35 | 12 | 0.679
quant_mobilenetv2_224_75 | 39 | 10 | 0.684
float_mnasnet_160_100 | 45 | 15 | 0.706
quant_mobilenetv2_224_100 | 48 | 12 | 0.704
float_mnasnet_224_75 | 55 | 18 | 0.718
float_mnasnet_192_100 | 62 | 20 | 0.724
float_mnasnet_224_100 | 84 | 27 | 0.742
float_mnasnet_224_130 | 126 | 40 | 0.758
float_v3-small-minimalistic_224_100 | - | 5 | 0.620
quant_v3-small_224_100 | - | 5 | 0.641
float_v3-small_224_75 | - | 5 | 0.656
float_v3-small_224_100 | - | 7 | 0.677
quant_v3-large_224_100 | - | 12 | 0.728
float_v3-large_224_75 | - | 15 | 0.735
float_v3-large-minimalistic_224_100 | - | 17 | 0.722
float_v3-large_224_100 | - | 20 | 0.753
### References
1. **NetAdapt: Platform-Aware Neural Network Adaptation for Mobile
Applications**<br />
Yang, Tien-Ju, Andrew Howard, Bo Chen, Xiao Zhang, Alec Go, Mark Sandler,
Vivienne Sze, and Hartwig Adam. In Proceedings of the European Conference
on Computer Vision (ECCV), pp. 285-300. 2018<br />
[[link]](https://arxiv.org/abs/1804.03230) arXiv:1804.03230, 2018.
@@ -0,0 +1,121 @@
<!--
• This is a README.md template we encourage you to use when you release your model.
• There are general sections we added to this template for various ML models.
• You may need to add or remove sections depending on your needs.
-->
# Project Name
## Authors
The **1st place winner** of the **4th On-device Visual Intelligence Competition** ([OVIC](https://docs.google.com/document/d/1Rxm_N7dGRyPXjyPIdRwdhZNRye52L56FozDnfYuCi0k/edit#)) of Low-Power Computer Vision Challenge ([LPCVC](https://lpcv.ai/))
* Last name, First name ([@GitHubUsername](https://github.com/username))
* Last name, First name ([@GitHubUsername](https://github.com/username))
* Last name, First name ([@GitHubUsername](https://github.com/username))
## Description
<!-- Provide description of the model -->
The model submitted for the OVIC and full implementation code for training, evaluation, and inference
* OVIC track: Image Classification, Object Detection
## Algorithm
<!-- Provide details of the algorithms used -->
## Requirements
<!--
• Provide description of the model
• Provide brief information of the algorithms used
-->
To install requirements:
```setup
pip install -r requirements.txt
```
## Pre-trained Models
| Model | Download | MD5 checksum |
|-------|----------|--------------|
| Model Name | Download Link (Size: KB) | MD5 checksum |
The model tar file contains the followings:
* Trained model checkpoint
* Frozen trained model
* TensorFlow Lite model
## Results
### [4th OVIC Public Ranked Leaderboard](https://lpcvc.ecn.purdue.edu/score_board_r4/?contest=round4)
#### Image Classification (from the Leaderboard)
| Rank | Username | Latency | Accuracy on Classified | # Classified | Accuracy/Time | Metric | Reference Accuracy |
|------|----------|---------|------------------------|--------------|---------------|--------|--------------------|
| 1 | Username | xx.x | 0.xxxx | 20000.0 | xxx | 0.xxxxx | 0.xxxxx |
* **Metric**: Accuracy improvement over the reference accuracy from the Pareto optimal curve
* **Accuracy on Classified**: The accuracy in [0, 1] computed based only on the images classified within the wall-time
* **\# Classified**: The number of images classified within the wall-time
* **Accuracy/Time**: The accuracy divided by either the total inference time or the wall-time, whichever is longer
* **Reference accuracy**: The reference accuracy of models from the Pareto optimal curve that have the same latency as the submission
#### Object Detection
| Rank | Username | Metric | Runtime | mAP over time | mAP of processed |
|------|----------|--------|---------|---------------|------------------|
| 1 | Username | 0.xxxxx | xxx.x | xxx | xxx |
* **Metric**: COCO mAP computed on the entire minival dataset
* **mAP over time**: COCO mAP on the minival dataset divided by latency per image
* **mAP of processed**: COCO mAP computed only on the processed images
## Dataset
<!--
• Provide detailed information of the dataset used
-->
## Training
<!--
• Provide detailed training information (preprocessing, hyperparameters, random seeds, and environment)
• Provide a command line example for training.
-->
Please run this command line for training.
```shell
python3 ...
```
## Evaluation
<!--
• Provide evaluation script with details of how to reproduce results.
• Describe data preprocessing / postprocessing steps
• Provide a command line example for evaluation.
-->
Please run this command line for evaluation.
```shell
python3 ...
```
## References
<!-- Link to references -->
## License
<!--
• Place your license text in a file named LICENSE.txt (or LICENSE.md) in the root of the repository.
• Please also include information about your license in this README.md file.
e.g., [Adding a license to a repository](https://help.github.com/en/github/building-a-strong-community/adding-a-license-to-a-repository)
-->
This project is licensed under the terms of the **Apache License 2.0**.
## Citation
<!--
If you want to make your repository citable, please follow the instructions at [Making Your Code Citable](https://guides.github.com/activities/citable-code/)
-->
If you want to cite this repository in your research paper, please use the following information.
@@ -0,0 +1,48 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
Copyright 2018 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="ovic.demo.app"
android:versionCode="1"
android:versionName="1.0" >
<uses-sdk
android:minSdkVersion="19"
android:targetSdkVersion="21" />
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
<uses-permission android:name="android.permission.READ_PHONE_STATE" />
<application
android:allowBackup="true"
android:icon="@drawable/ic_launcher"
android:largeHeap="true"
android:label="@string/app_name">
<activity
android:name="ovic.demo.app.OvicBenchmarkerActivity"
android:label="@string/app_name"
android:screenOrientation="portrait">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity>
</application>
</manifest>
+34
View File
@@ -0,0 +1,34 @@
# Sample app for OVIC benchmarking.
load("@build_bazel_rules_android//android:rules.bzl", "android_binary")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
licenses = ["notice"],
)
android_binary(
name = "ovic_benchmarker_binary",
srcs = [
"OvicBenchmarkerActivity.java",
],
assets = [
"//tensorflow/lite/java/ovic/src/testdata:coco_labels.txt",
"//tensorflow/lite/java/ovic/src/testdata:labels.txt",
"//tensorflow/lite/java/ovic/src/testdata:ovic_testdata",
],
assets_dir = "",
custom_package = "ovic.demo.app",
manifest = "AndroidManifest.xml",
nocompress_extensions = [
".lite",
".tflite",
],
resource_files = glob(["res/**"]),
tags = ["manual"],
deps = [
"//tensorflow/lite/java:tensorflowlite",
"//tensorflow/lite/java/ovic:ovicbenchmarkerlib",
"//tensorflow/lite/java/ovic:ovicdetectionbenchmarkerlib",
],
)
@@ -0,0 +1,279 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package ovic.demo.app;
import android.app.Activity;
import android.content.res.AssetFileDescriptor;
import android.content.res.AssetManager;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.os.Bundle;
import android.os.Process;
import android.os.SystemClock;
import android.util.Log;
import android.view.View;
import android.widget.TextView;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.text.DecimalFormat;
import org.tensorflow.ovic.OvicBenchmarker;
import org.tensorflow.ovic.OvicClassifierBenchmarker;
import org.tensorflow.ovic.OvicDetectorBenchmarker;
/** Class that benchmark image classifier models. */
public class OvicBenchmarkerActivity extends Activity {
/** Tag for the {@link Log}. */
private static final String TAG = "OvicBenchmarkerActivity";
/** Name of the task-dependent data files stored in Assets. */
private static String labelPath = null;
private static String testImagePath = null;
private static String modelPath = null;
/**
* Each bottom press will launch a benchmarking experiment. The experiment stops when either the
* total native latency reaches WALL_TIME or the number of iterations reaches MAX_ITERATIONS,
* whichever comes first.
*/
/** Wall time for each benchmarking experiment. */
private static final double WALL_TIME = 3000;
/** Maximum number of iterations in each benchmarking experiment. */
private static final int MAX_ITERATIONS = 100;
/** Mask for binding to a single big core. Pixel 1 (4), Pixel 2 (16). */
private static final int BIG_CORE_MASK = 16;
/** Amount of time in milliseconds to wait for affinity to set. */
private static final int WAIT_TIME_FOR_AFFINITY = 1000;
/* The model to be benchmarked. */
private MappedByteBuffer model = null;
private InputStream labelInputStream = null;
private OvicBenchmarker benchmarker;
private TextView textView = null;
// private Button startButton = null;
private static final DecimalFormat df2 = new DecimalFormat(".##");
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
// TextView used to display the progress, for information purposes only.
textView = (TextView) findViewById(R.id.textView);
}
private Bitmap loadTestBitmap() throws IOException {
InputStream imageStream = getAssets().open(testImagePath);
return BitmapFactory.decodeStream(imageStream);
}
public void initializeTest(boolean benchmarkClassification) throws IOException {
Log.i(TAG, "Initializing benchmarker.");
if (benchmarkClassification) {
benchmarker = new OvicClassifierBenchmarker(WALL_TIME);
labelPath = "labels.txt";
testImagePath = "test_image_224.jpg";
modelPath = "quantized_model.lite";
} else { // Benchmarking detection.
benchmarker = new OvicDetectorBenchmarker(WALL_TIME);
labelPath = "coco_labels.txt";
testImagePath = "test_image_224.jpg";
modelPath = "detect.lite";
}
AssetManager am = getAssets();
AssetFileDescriptor fileDescriptor = am.openFd(modelPath);
FileInputStream modelInputStream = new FileInputStream(fileDescriptor.getFileDescriptor());
FileChannel fileChannel = modelInputStream.getChannel();
long startOffset = fileDescriptor.getStartOffset();
long declaredLength = fileDescriptor.getDeclaredLength();
model = fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
labelInputStream = am.open(labelPath);
}
public Boolean doTestIteration() throws IOException, InterruptedException {
if (benchmarker == null) {
throw new RuntimeException("Benchmarker has not been initialized.");
}
if (benchmarker.shouldStop()) {
return false;
}
if (!benchmarker.readyToTest()) {
Log.i(TAG, "getting ready to test.");
benchmarker.getReadyToTest(labelInputStream, model);
if (!benchmarker.readyToTest()) {
throw new RuntimeException("Failed to get the benchmarker ready.");
}
}
Log.i(TAG, "Going to do test iter.");
// Start testing.
Bitmap testImageBitmap = loadTestBitmap();
try {
if (!benchmarker.processBitmap(testImageBitmap)) {
throw new RuntimeException("Failed to run test.");
}
} catch (Exception e) {
e.printStackTrace();
throw e;
} finally {
testImageBitmap.recycle();
}
String iterResultString = benchmarker.getLastResultString();
if (iterResultString == null) {
throw new RuntimeException("Inference failed to produce a result.");
}
Log.i(TAG, iterResultString);
return true;
}
public void detectPressed(View view) throws IOException {
benchmarkSession(false);
}
public void classifyPressed(View view) throws IOException {
benchmarkSession(true);
}
private void benchmarkSession(boolean benchmarkClassification) throws IOException {
try {
initializeTest(benchmarkClassification);
} catch (IOException e) {
Log.e(TAG, "Can't initialize benchmarker.", e);
throw e;
}
String displayText = "";
if (benchmarkClassification) {
displayText = "Classification benchmark: ";
} else {
displayText = "Detection benchmark: ";
}
try {
setProcessorAffinity(BIG_CORE_MASK);
} catch (IOException e) {
Log.e(TAG, e.getMessage());
displayText = e.getMessage() + "\n";
}
Log.i(TAG, "Successfully initialized benchmarker.");
int testIter = 0;
Boolean iterSuccess = false;
while (testIter < MAX_ITERATIONS) {
try {
iterSuccess = doTestIteration();
} catch (IOException e) {
Log.e(TAG, "Error during iteration " + testIter);
throw e;
} catch (InterruptedException e) {
Log.e(TAG, "Interrupted at iteration " + testIter);
displayText += e.getMessage() + "\n";
}
if (!iterSuccess) {
break;
}
testIter++;
}
Log.i(TAG, "Benchmarking finished");
if (textView != null) {
if (testIter > 0) {
textView.setText(
displayText
+ modelPath
+ ": Average latency="
+ df2.format(benchmarker.getTotalRuntimeNano() * 1.0e-6 / testIter)
+ "ms after "
+ testIter
+ " runs.");
} else {
textView.setText("Benchmarker failed to run on more than one images.");
}
}
}
// TODO(b/153429929) Remove with resolution of issue (see below).
@SuppressWarnings("RuntimeExec")
private static void setProcessorAffinity(int mask) throws IOException {
int myPid = Process.myPid();
Log.i(TAG, String.format("Setting processor affinity to 0x%02x", mask));
String command = String.format("taskset -a -p %x %d", mask, myPid);
try {
// TODO(b/153429929) This is deprecated, but updating is not safe while verification is hard.
Runtime.getRuntime().exec(command).waitFor();
} catch (InterruptedException e) {
throw new IOException("Interrupted: " + e);
}
// Make sure set took effect - try for a second to confirm the change took. If not then fail.
long startTimeMs = SystemClock.elapsedRealtime();
while (true) {
int readBackMask = readCpusAllowedMask();
if (readBackMask == mask) {
Log.i(TAG, String.format("Successfully set affinity to 0x%02x", mask));
return;
}
if (SystemClock.elapsedRealtime() > startTimeMs + WAIT_TIME_FOR_AFFINITY) {
throw new IOException(
String.format(
"Core-binding failed: affinity set to 0x%02x but read back as 0x%02x\n"
+ "please root device.",
mask, readBackMask));
}
try {
Thread.sleep(50);
} catch (InterruptedException e) {
// Ignore sleep interrupted, will sleep again and compare is final cross-check.
}
}
}
public static int readCpusAllowedMask() throws IOException {
// Determine how many CPUs there are total
final String pathname = "/proc/self/status";
final String resultPrefix = "Cpus_allowed:";
File file = new File(pathname);
String line = "<NO LINE READ>";
String allowedCPU = "";
Integer allowedMask = null;
BufferedReader bufReader = null;
try {
bufReader = new BufferedReader(new FileReader(file));
while ((line = bufReader.readLine()) != null) {
if (line.startsWith(resultPrefix)) {
allowedMask = Integer.valueOf(line.substring(resultPrefix.length()).trim(), 16);
allowedCPU = bufReader.readLine();
break;
}
}
} catch (RuntimeException e) {
throw new IOException(
"Invalid number in " + pathname + " line: \"" + line + "\": " + e.getMessage());
} finally {
if (bufReader != null) {
bufReader.close();
}
}
if (allowedMask == null) {
throw new IOException(pathname + " missing " + resultPrefix + " line");
}
Log.i(TAG, allowedCPU);
return allowedMask;
}
}
@@ -0,0 +1,49 @@
apply plugin: 'com.android.application'
android {
compileSdkVersion 26
buildToolsVersion "27.0.3"
defaultConfig {
applicationId "android.example.com.ovicbenchmarker"
minSdkVersion 15
targetSdkVersion 26
versionCode 1
versionName "1.0"
}
lintOptions {
abortOnError false
}
buildTypes {
release {
minifyEnabled false
proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
}
}
aaptOptions {
noCompress "lite", "tflite"
}
compileOptions {
sourceCompatibility JavaVersion.VERSION_1_8
targetCompatibility JavaVersion.VERSION_1_8
}
}
repositories {
mavenCentral()
maven { // Only for snapshot artifacts
name 'ossrh-snapshot'
url 'https://oss.sonatype.org/content/repositories/snapshots'
}
}
dependencies {
implementation fileTree(dir: 'libs', include: ['*.jar'])
implementation 'com.android.support:appcompat-v7:25.2.0'
implementation 'com.android.support.constraint:constraint-layout:1.0.2'
implementation 'com.android.support:design:25.2.0'
implementation 'com.android.support:support-annotations:25.3.1'
implementation 'com.android.support:support-v13:25.2.0'
implementation 'org.tensorflow:tensorflow-lite:+'
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 2.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.1 KiB

@@ -0,0 +1,39 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
Copyright 2018 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<selector
xmlns:android="http://schemas.android.com/apk/res/android">
<item
android:state_enabled="false">
<shape android:shape="rectangle">
<solid android:color="#808080"/>
</shape>
</item>
<item
android:state_enabled="true"
android:state_pressed="true">
<shape android:shape="rectangle">
<solid android:color="#44ff44"/>
</shape>
</item>
<item
android:state_enabled="true"
android:state_pressed="false">
<shape android:shape="rectangle" >
<solid android:color="#227f22"/>
</shape>
</item>
</selector>
@@ -0,0 +1,73 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
Copyright 2018 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<RelativeLayout
xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:paddingBottom="@dimen/activity_vertical_margin"
android:paddingLeft="@dimen/activity_horizontal_margin"
android:paddingRight="@dimen/activity_horizontal_margin"
android:paddingTop="@dimen/activity_vertical_margin"
tools:context="ovic.demo.app.OvicBenchmarkerActivity">
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/initial_status_msg"
android:id="@+id/textView"
android:layout_above="@+id/button_clf_start"
android:layout_alignParentTop="true"/>
<Button
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/start_clf_label"
android:id="@id/button_clf_start"
android:layout_alignParentBottom="true"
android:layout_alignParentLeft="true"
android:background="@drawable/start_button_color"
android:padding="10dp"
android:layout_marginRight="30dp"
android:layout_marginLeft="100dp"
android:layout_marginTop="10dp"
android:foreground="#000000"
android:textColor="#ffffff"
android:enabled="true"
style="?android:attr/buttonBarButtonStyle"
android:onClick="classifyPressed"/>
<Button
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/start_det_label"
android:id="@+id/button_det_start"
android:layout_alignParentBottom="true"
android:layout_alignParentRight="true"
android:layout_toRightOf="@id/button_clf_start"
android:background="@drawable/start_button_color"
android:padding="10dp"
android:layout_marginRight="100dp"
android:layout_marginLeft="30dp"
android:layout_marginTop="10dp"
android:foreground="#000000"
android:textColor="#ffffff"
android:enabled="true"
style="?android:attr/buttonBarButtonStyle"
android:onClick="detectPressed"/>
</RelativeLayout>
@@ -0,0 +1,20 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
Copyright 2018 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources>
<dimen name="activity_vertical_margin">20dp</dimen>
<dimen name="activity_horizontal_margin">16dp</dimen>
</resources>
@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
Copyright 2018 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources>
<string name="app_name" translatable="false">Benchmarker</string>
<string name="start_clf_label" translatable="false">Clf</string>
<string name="start_det_label" translatable="false">Det</string>
<string name="initial_status_msg" translatable="false"> Press start to run the benchmarks.</string>
</resources>
@@ -0,0 +1,29 @@
// Top-level build file where you can add configuration options common to all sub-projects/modules.
buildscript {
repositories {
google()
mavenCentral()
}
dependencies {
classpath 'com.android.tools.build:gradle:3.1.4'
// NOTE: Do not place your application dependencies here; they belong
// in the individual module build.gradle files
}
}
allprojects {
repositories {
google()
mavenCentral()
maven { // Only for snapshot artifacts
name 'ossrh-snapshot'
url 'https://oss.sonatype.org/content/repositories/snapshots'
}
}
}
task clean(type: Delete) {
delete rootProject.buildDir
}
@@ -0,0 +1,17 @@
# Project-wide Gradle settings.
# IDE (e.g. Android Studio) users:
# Gradle settings configured through the IDE *will override*
# any settings specified in this file.
# For more details on how to configure your build environment visit
# http://www.gradle.org/docs/current/userguide/build_environment.html
# Specifies the JVM arguments used for the daemon process.
# The setting is particularly useful for tweaking memory settings.
org.gradle.jvmargs=-Xmx1536m
# When configured, Gradle will run in incubating parallel mode.
# This option should only be used with decoupled projects. More details, visit
# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
# org.gradle.parallel=true
Binary file not shown.
@@ -0,0 +1,6 @@
#Thu Sep 28 09:01:41 PDT 2017
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-all.zip
+160
View File
@@ -0,0 +1,160 @@
#!/usr/bin/env bash
##############################################################################
##
## Gradle start up script for UN*X
##
##############################################################################
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS=""
APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
warn ( ) {
echo "$*"
}
die ( ) {
echo
echo "$*"
echo
exit 1
}
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
case "`uname`" in
CYGWIN* )
cygwin=true
;;
Darwin* )
darwin=true
;;
MINGW* )
msys=true
;;
esac
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
PRG="$0"
# Need this for relative symlinks.
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "$PRG"`"/$link"
fi
done
SAVED="`pwd`"
cd "`dirname \"$PRG\"`/" >/dev/null
APP_HOME="`pwd -P`"
cd "$SAVED" >/dev/null
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD="java"
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
MAX_FD_LIMIT=`ulimit -H -n`
if [ $? -eq 0 ] ; then
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
MAX_FD="$MAX_FD_LIMIT"
fi
ulimit -n $MAX_FD
if [ $? -ne 0 ] ; then
warn "Could not set maximum file descriptor limit: $MAX_FD"
fi
else
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
fi
fi
# For Darwin, add options to specify how the application appears in the dock
if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi
# For Cygwin, switch paths to Windows format before running java
if $cygwin ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
JAVACMD=`cygpath --unix "$JAVACMD"`
# We build the pattern for arguments to be converted via cygpath
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
SEP=""
for dir in $ROOTDIRSRAW ; do
ROOTDIRS="$ROOTDIRS$SEP$dir"
SEP="|"
done
OURCYGPATTERN="(^($ROOTDIRS))"
# Add a user-defined pattern to the cygpath arguments
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
fi
# Now convert the arguments - kludge to limit ourselves to /bin/sh
i=0
for arg in "$@" ; do
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
else
eval `echo args$i`="\"$arg\""
fi
i=$((i+1))
done
case $i in
(0) set -- ;;
(1) set -- "$args0" ;;
(2) set -- "$args0" "$args1" ;;
(3) set -- "$args0" "$args1" "$args2" ;;
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi
# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
function splitJvmOpts() {
JVM_OPTS=("$@")
}
eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+105
View File
@@ -0,0 +1,105 @@
:: Copyright 2026 The TensorFlow Authors. All Rights Reserved.
::
:: Licensed under the Apache License, Version 2.0 (the "License");
:: you may not use this file except in compliance with the License.
:: You may obtain a copy of the License at
::
:: http://www.apache.org/licenses/LICENSE-2.0
::
:: Unless required by applicable law or agreed to in writing, software
:: distributed under the License is distributed on an "AS IS" BASIS,
:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
:: See the License for the specific language governing permissions and
:: limitations under the License.
:: ==============================================================================
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@rem Gradle startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables with windows NT shell
if "%OS%"=="Windows_NT" setlocal
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS=
set DIRNAME=%~dp0
if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if "%ERRORLEVEL%" == "0" goto init
echo.
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto init
echo.
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:init
@rem Get command-line arguments, handling Windowz variants
if not "%OS%" == "Windows_NT" goto win9xME_args
if "%@eval[2+2]" == "4" goto 4NT_args
:win9xME_args
@rem Slurp the command line arguments.
set CMD_LINE_ARGS=
set _SKIP=2
:win9xME_args_slurp
if "x%~1" == "x" goto execute
set CMD_LINE_ARGS=%*
goto execute
:4NT_args
@rem Get arguments from the 4NT Shell from JP Software
set CMD_LINE_ARGS=%$
:execute
@rem Setup the command line
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
:end
@rem End local scope for the variables with windows NT shell
if "%ERRORLEVEL%"=="0" goto mainEnd
:fail
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
rem the _cmd.exe /c_ return code!
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
exit /b 1
:mainEnd
if "%OS%"=="Windows_NT" endlocal
:omega
@@ -0,0 +1 @@
include ':app'
@@ -0,0 +1,68 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.ovic;
/** Class for holding a detection bounding box with category and confidence. */
public class BoundingBox {
// Upper left point.
public float x1;
public float y1;
// Lower right point.
public float x2;
public float y2;
// The area of the box
public float area;
// The object category
public int category;
// The confidence of the detection
public float score;
public BoundingBox(float x1, float y1, float x2, float y2, int category, float score) {
this.x1 = x1;
this.y1 = y1;
this.x2 = x2;
this.y2 = y2;
this.category = category;
this.score = score;
// -1 stands for area not initialized
this.area = -1;
}
// The intersection area of two bounding boxes
public float intersect(BoundingBox bbx) {
return Math.max(0, Math.min(x2, bbx.x2) - Math.max(x1, bbx.x1))
* Math.max(0, Math.min(y2, bbx.y2) - Math.max(y1, bbx.y1));
}
// The union area of two bounding boxes
public float union(BoundingBox bbx) {
return bbx.getArea() + this.getArea() - this.intersect(bbx);
}
public float getArea() {
if (area < 0) {
area = (x2 - x1) * (y2 - y1);
}
return area;
}
public float computeIoU(BoundingBox bbx) {
return (float) (this.intersect(bbx) * 1.0 / this.union(bbx));
}
}
@@ -0,0 +1,147 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.ovic;
import android.graphics.Bitmap;
import android.os.SystemClock;
import android.util.Log;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
/**
* Base class that benchmarks image models.
*
* <p>===================== General workflow =======================
*
* <pre>{@code
* benchmarker = new OvicBenchmarker();
* benchmarker.getReadyToTest(labelInputStream, model);
* while (!benchmarker.shouldStop()) {
* Bitmap bitmap = ...
* imgId = ...
* benchmarker.processBitmap(bitmap, imgId);
* }
* }</pre>
*/
public abstract class OvicBenchmarker {
/** Tag for the {@link Log}. */
private static final String TAG = "OvicBenchmarker";
/** Dimensions of inputs. */
protected static final int DIM_BATCH_SIZE = 1;
protected static final int DIM_PIXEL_SIZE = 3;
protected int imgHeight = 224;
protected int imgWidth = 224;
/* Preallocated buffers for storing image data in. */
protected int[] intValues = null;
/** A ByteBuffer to hold image data, to be feed into classifier as inputs. */
protected ByteBuffer imgData = null;
/** Total runtime in ns. */
protected double totalRuntimeNano = 0.0;
/** Total allowed runtime in ms. */
protected double wallTimeMilli = 20000 * 30.0;
/** Record whether benchmark has started (used to skip the first image). */
protected boolean benchmarkStarted = false;
/**
* Initializes an {@link OvicBenchmarker}
*
* @param wallTimeMilli: a double number specifying the total amount of time to benchmark.
*/
protected OvicBenchmarker(double wallTimeMilli) {
benchmarkStarted = false;
totalRuntimeNano = 0.0;
this.wallTimeMilli = wallTimeMilli;
}
/** Return the cumulative latency of all runs so far. */
public double getTotalRuntimeNano() {
return totalRuntimeNano;
}
/** Check whether the benchmarker should stop. */
public Boolean shouldStop() {
if ((totalRuntimeNano * 1.0 / 1e6) >= wallTimeMilli) {
Log.e(
TAG,
"Total runtime "
+ (totalRuntimeNano * 1.0 / 1e6)
+ " exceeded walltime (ms) "
+ wallTimeMilli);
return true;
}
return false;
}
/** Abstract class for checking whether the benchmarker is ready to start processing images */
public abstract boolean readyToTest();
/**
* Abstract class for getting the benchmarker ready.
*
* @param labelInputStream: an {@link InputStream} specifying where the list of labels should be
* read from.
* @param model: a {@link MappedByteBuffer} model to benchmark.
*/
public abstract void getReadyToTest(InputStream labelInputStream, MappedByteBuffer model);
/**
* Perform test on a single bitmap image.
*
* @param bitmap: a {@link Bitmap} image to process.
* @param imageId: an ID uniquely representing the image.
*/
public abstract boolean processBitmap(Bitmap bitmap, int imageId)
throws IOException, InterruptedException;
/** Perform test on a single bitmap image without an image ID. */
public boolean processBitmap(Bitmap bitmap) throws IOException, InterruptedException {
return processBitmap(bitmap, /* imageId = */ 0);
}
/** Returns the last inference results as string. */
public abstract String getLastResultString();
/**
* Loads input buffer from intValues into ByteBuffer for the interpreter. Input buffer must be
* loaded in intValues and output will be placed in imgData.
*/
protected void loadsInputToByteBuffer() {
if (imgData == null || intValues == null) {
throw new RuntimeException("Benchmarker is not yet ready to test.");
}
// Convert the image to ByteBuffer.
imgData.rewind();
int pixel = 0;
long startTime = SystemClock.uptimeMillis();
for (int i = 0; i < imgHeight; ++i) {
for (int j = 0; j < imgWidth; ++j) {
final int pixelValue = intValues[pixel++];
imgData.put((byte) ((pixelValue >> 16) & 0xFF));
imgData.put((byte) ((pixelValue >> 8) & 0xFF));
imgData.put((byte) (pixelValue & 0xFF));
}
}
long endTime = SystemClock.uptimeMillis();
Log.d(TAG, "Timecost to put values into ByteBuffer: " + Long.toString(endTime - startTime));
}
}
@@ -0,0 +1,59 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.ovic;
import java.util.ArrayList;
/** Result class for inference run on a single image. */
public class OvicClassificationResult {
/** Top K classes and probabilities. */
public final ArrayList<String> topKClasses;
public final ArrayList<Float> topKProbs;
public final ArrayList<Integer> topKIndices;
/** Latency (ms). */
public Long latencyMilli;
/** Latency (ns). */
public Long latencyNano;
OvicClassificationResult() {
topKClasses = new ArrayList<>();
topKProbs = new ArrayList<>();
topKIndices = new ArrayList<>();
latencyMilli = -1L;
latencyNano = -1L;
}
@Override
public String toString() {
String textToShow = latencyMilli + "ms";
textToShow += "\n" + latencyNano + "ns";
for (int k = 0; k < topKProbs.size(); ++k) {
textToShow +=
"\nPrediction ["
+ k
+ "] = Class "
+ topKIndices.get(k)
+ " ("
+ topKClasses.get(k)
+ ") : "
+ topKProbs.get(k);
}
return textToShow;
}
}
@@ -0,0 +1,227 @@
/*Copyright 2018 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.ovic;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
import org.tensorflow.lite.Interpreter;
import org.tensorflow.lite.TestHelper;
/** Class for running ImageNet classification with a TfLite model. */
public class OvicClassifier {
/** Tag for the {@link Log}. */
private static final String TAG = "OvicClassifier";
/** Number of results to show (i.e. the "K" in top-K predictions). */
private static final int RESULTS_TO_SHOW = 5;
/** An instance of the driver class to run model inference with Tensorflow Lite. */
private Interpreter tflite;
/** Labels corresponding to the output of the vision model. */
private final List<String> labelList;
/** An array to hold inference results, to be feed into Tensorflow Lite as outputs. */
private byte[][] inferenceOutputArray = null;
/** An array to hold final prediction probabilities. */
private float[][] labelProbArray = null;
/** Input resultion. */
private int[] inputDims = null;
/** Whether the model runs as float or quantized. */
private Boolean outputIsFloat = null;
private final PriorityQueue<Map.Entry<Integer, Float>> sortedLabels =
new PriorityQueue<>(
RESULTS_TO_SHOW,
new Comparator<Map.Entry<Integer, Float>>() {
@Override
public int compare(Map.Entry<Integer, Float> o1, Map.Entry<Integer, Float> o2) {
return o1.getValue().compareTo(o2.getValue());
}
});
/** Initializes an {@code OvicClassifier}. */
public OvicClassifier(InputStream labelInputStream, MappedByteBuffer model) throws IOException {
if (model == null) {
throw new RuntimeException("Input model is empty.");
}
labelList = loadLabelList(labelInputStream);
// OVIC uses one thread for CPU inference.
tflite = new Interpreter(model, new Interpreter.Options().setNumThreads(1));
inputDims = TestHelper.getInputDims(tflite, 0);
if (inputDims.length != 4) {
throw new RuntimeException("The model's input dimensions must be 4 (BWHC).");
}
if (inputDims[0] != 1) {
throw new IllegalStateException(
"The model must have a batch size of 1, got " + inputDims[0] + " instead.");
}
if (inputDims[3] != 3) {
throw new IllegalStateException(
"The model must have three color channels, got " + inputDims[3] + " instead.");
}
int minSide = Math.min(inputDims[1], inputDims[2]);
int maxSide = Math.max(inputDims[1], inputDims[2]);
if (minSide <= 0 || maxSide > 1000) {
throw new RuntimeException("The model's resolution must be between (0, 1000].");
}
String outputDataType = TestHelper.getOutputDataType(tflite, 0);
switch (outputDataType) {
case "float":
outputIsFloat = true;
break;
case "byte":
outputIsFloat = false;
break;
default:
throw new IllegalStateException("Cannot process output type: " + outputDataType);
}
inferenceOutputArray = new byte[1][labelList.size()];
labelProbArray = new float[1][labelList.size()];
}
/** Classifies a {@link ByteBuffer} image. */
// @throws RuntimeException if model is uninitialized.
public OvicClassificationResult classifyByteBuffer(ByteBuffer imgData) {
if (tflite == null) {
throw new RuntimeException(TAG + ": ImageNet classifier has not been initialized; Failed.");
}
if (outputIsFloat == null) {
throw new RuntimeException(TAG + ": Classifier output type has not been resolved.");
}
if (outputIsFloat) {
tflite.run(imgData, labelProbArray);
} else {
tflite.run(imgData, inferenceOutputArray);
/** Convert results to float */
for (int i = 0; i < inferenceOutputArray[0].length; i++) {
labelProbArray[0][i] = (inferenceOutputArray[0][i] & 0xff) / 255.0f;
}
}
OvicClassificationResult iterResult = computeTopKLabels();
iterResult.latencyMilli = getLastNativeInferenceLatencyMilliseconds();
iterResult.latencyNano = getLastNativeInferenceLatencyNanoseconds();
return iterResult;
}
/** Return the probability array of all classes. */
public float[][] getlabelProbArray() {
return labelProbArray;
}
/** Return the number of top labels predicted by the classifier. */
public int getNumPredictions() {
return RESULTS_TO_SHOW;
}
/** Return the four dimensions of the input image. */
public int[] getInputDims() {
return inputDims;
}
/**
* Get native inference latency of last image classification run.
*
* @throws RuntimeException if model is uninitialized.
*/
public Long getLastNativeInferenceLatencyMilliseconds() {
if (tflite == null) {
throw new RuntimeException(TAG + ": ImageNet classifier has not been initialized; Failed.");
}
Long latency = tflite.getLastNativeInferenceDurationNanoseconds();
return (latency == null) ? null : (Long) (latency / 1000000);
}
/**
* Get native inference latency of last image classification run.
*
* @throws RuntimeException if model is uninitialized.
*/
public Long getLastNativeInferenceLatencyNanoseconds() {
if (tflite == null) {
throw new IllegalStateException(
TAG + ": ImageNet classifier has not been initialized; Failed.");
}
return tflite.getLastNativeInferenceDurationNanoseconds();
}
/** Closes tflite to release resources. */
public void close() {
tflite.close();
tflite = null;
}
/** Reads label list from Assets. */
private static List<String> loadLabelList(InputStream labelInputStream) throws IOException {
List<String> labelList = new ArrayList<>();
try (BufferedReader reader =
new BufferedReader(new InputStreamReader(labelInputStream, UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
labelList.add(line);
}
}
return labelList;
}
/** Computes top-K labels. */
private OvicClassificationResult computeTopKLabels() {
if (labelList == null) {
throw new RuntimeException("Label file has not been loaded.");
}
for (int i = 0; i < labelList.size(); ++i) {
sortedLabels.add(new AbstractMap.SimpleEntry<>(i, labelProbArray[0][i]));
if (sortedLabels.size() > RESULTS_TO_SHOW) {
sortedLabels.poll();
}
}
OvicClassificationResult singleImageResult = new OvicClassificationResult();
if (sortedLabels.size() != RESULTS_TO_SHOW) {
throw new RuntimeException(
"Number of returned labels does not match requirement: "
+ sortedLabels.size()
+ " returned, but "
+ RESULTS_TO_SHOW
+ " required.");
}
for (int i = 0; i < RESULTS_TO_SHOW; ++i) {
Map.Entry<Integer, Float> label = sortedLabels.poll();
// ImageNet model prediction indices are 0-based.
singleImageResult.topKIndices.add(label.getKey());
singleImageResult.topKClasses.add(labelList.get(label.getKey()));
singleImageResult.topKProbs.add(label.getValue());
}
// Labels with lowest probability are returned first, hence need to reverse them.
Collections.reverse(singleImageResult.topKIndices);
Collections.reverse(singleImageResult.topKClasses);
Collections.reverse(singleImageResult.topKProbs);
return singleImageResult;
}
}
@@ -0,0 +1,142 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.ovic;
import android.graphics.Bitmap;
import android.util.Log;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
/** Class that benchmarks image classifier models. */
public final class OvicClassifierBenchmarker extends OvicBenchmarker {
/** Tag for the {@link Log}. */
private static final String TAG = "OvicClassifierBenchmarker";
/** ImageNet preprocessing parameters. */
private static final float CENTRAL_FRACTION = 0.875f;
private OvicClassifier classifier;
private OvicClassificationResult iterResult = null;
public OvicClassifierBenchmarker(double wallTime) {
super(wallTime);
}
/** Test if the classifier is ready for benchmarking. */
@Override
public boolean readyToTest() {
return (classifier != null);
}
/**
* Getting the benchmarker ready for classifying images.
*
* @param labelInputStream: an {@link InputStream} specifying where the list of labels should be
* read from.
* @param model: a {@link MappedByteBuffer} model to benchmark.
*/
@Override
public void getReadyToTest(InputStream labelInputStream, MappedByteBuffer model) {
try {
Log.i(TAG, "Creating classifier.");
classifier = new OvicClassifier(labelInputStream, model);
int [] inputDims = classifier.getInputDims();
imgHeight = inputDims[1];
imgWidth = inputDims[2];
// Only accept QUANTIZED_UINT8 input.
imgData = ByteBuffer.allocateDirect(DIM_BATCH_SIZE * imgHeight * imgWidth * DIM_PIXEL_SIZE);
imgData.order(ByteOrder.nativeOrder());
intValues = new int[imgHeight * imgWidth];
} catch (Exception e) {
Log.e(TAG, e.getMessage());
Log.e(TAG, "Failed to initialize ImageNet classifier for the benchmarker.");
}
}
/**
* Perform classification on a single bitmap image.
*
* @param bitmap: a {@link Bitmap} image to process.
* @param imageId: an ID uniquely representing the image.
*/
@Override
public boolean processBitmap(Bitmap bitmap, int imageId)
throws IOException, InterruptedException {
if (shouldStop() || !readyToTest()) {
return false;
}
try {
Log.i(TAG, "Converting bitmap.");
convertBitmapToInput(bitmap);
Log.i(TAG, "Classifying image: " + imageId);
iterResult = classifier.classifyByteBuffer(imgData);
} catch (RuntimeException e) {
Log.e(TAG, e.getMessage());
Log.e(TAG, "Failed to classify image.");
}
if (iterResult == null || iterResult.latencyMilli == null || iterResult.latencyNano == null) {
throw new RuntimeException("Classification result or timing is invalid.");
}
Log.d(TAG, "Native inference latency (ms): " + iterResult.latencyMilli);
Log.d(TAG, "Native inference latency (ns): " + iterResult.latencyNano);
Log.i(TAG, iterResult.toString());
if (!benchmarkStarted) { // Skip the first image to discount warming-up time.
benchmarkStarted = true;
} else {
totalRuntimeNano += ((double) iterResult.latencyNano);
}
return true;
}
/** Return how many classes are predicted per image. */
public int getNumPredictions() {
return classifier.getNumPredictions();
}
public OvicClassificationResult getLastClassificationResult() {
return iterResult;
}
@Override
public String getLastResultString() {
if (iterResult == null) {
return null;
} else {
return iterResult.toString();
}
}
/**
* Preprocess bitmap according to ImageNet protocol then writes result into a {@link ByteBuffer}.
*
* @param bitmap: a {@link Bitmap} source image.
*/
private void convertBitmapToInput(Bitmap bitmap) {
// Perform transformations corresponding to evaluation mode.
float width = (float) bitmap.getWidth();
float height = (float) bitmap.getHeight();
int stWidth = Math.round((width - width * CENTRAL_FRACTION) / 2);
int stHeight = Math.round((height - height * CENTRAL_FRACTION) / 2);
int newWidth = Math.round(width - stWidth * 2);
int newHeight = Math.round(height - stHeight * 2);
bitmap = Bitmap.createBitmap(bitmap, stWidth, stHeight, newWidth, newHeight);
bitmap = Bitmap.createScaledBitmap(bitmap, imgWidth, imgHeight, true);
bitmap.getPixels(intValues, 0, bitmap.getWidth(), 0, 0, bitmap.getWidth(), bitmap.getHeight());
loadsInputToByteBuffer();
}
}
@@ -0,0 +1,95 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.ovic;
import java.util.ArrayList;
/** Result class for inference run on a single image. */
public class OvicDetectionResult {
// Top K classes and probabilities.
public final ArrayList<BoundingBox> detections;
// Latency (ms).
public Long latencyMilli = -1L;
// Latency (ns).
public Long latencyNano = -1L;
// id of the image.
public int id = -1;
// Number of valid detections (separately maintained, maybe different from detections.size()).
public int count = 0;
// Create OvicDetectionResult object with pre-filled capacity. Note that detections.size() will
// be equal to capacity after this call.
OvicDetectionResult(int capacity) {
detections = new ArrayList<BoundingBox>(capacity);
for (int i = 0; i < capacity; i++) {
detections.add(new BoundingBox(-1.0f, -1.0f, -1.0f, -1.0f, -1, -1.0f));
}
}
public void resetTo(Long latencyMilli, Long latencyNano, int id) {
count = 0;
this.latencyMilli = latencyMilli;
this.latencyNano = latencyNano;
this.id = id;
}
public void addBox(float x1, float y1, float x2, float y2, int category, float score) {
detections.get(count).x1 = x1;
detections.get(count).y1 = y1;
detections.get(count).x2 = x2;
detections.get(count).y2 = y2;
detections.get(count).category = category;
detections.get(count).score = score;
count += 1;
}
public void scaleUp(double scaleFactorWidth, double scaleFactorHeight) {
for (BoundingBox box : detections) {
box.x1 = (float) (box.x1 * scaleFactorWidth);
box.y1 = (float) (box.y1 * scaleFactorHeight);
box.x2 = (float) (box.x2 * scaleFactorWidth);
box.y2 = (float) (box.y2 * scaleFactorHeight);
}
}
@Override
public String toString() {
String textToShow = latencyMilli + "ms";
textToShow += "\n" + latencyNano + "ns";
int k = 0;
for (BoundingBox box : detections) {
textToShow +=
"\nPrediction ["
+ k
+ "] = Class "
+ box.category
+ " ("
+ box.x1
+ ", "
+ box.y1
+ ", "
+ box.x2
+ ", "
+ box.y2
+ ") : "
+ box.score;
k++;
}
return textToShow;
}
}
@@ -0,0 +1,198 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.ovic;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.tensorflow.lite.Interpreter;
import org.tensorflow.lite.TestHelper;
/** Class for running COCO detection with a TfLite model. */
public class OvicDetector implements AutoCloseable {
/** Tag for the {@link Log}. */
private static final String TAG = "OvicDetector";
/** An instance of the driver class to run model inference with Tensorflow Lite. */
private Interpreter tflite;
/** Labels corresponding to the output of the vision model. */
private final List<String> labelList;
/** Number of detections per image. 10 for demo, 100 for the actual competition. */
private static final int NUM_RESULTS = 100;
/** The output arrays for the mobilenet SSD. */
private float[][][] outputLocations;
private float[][] outputClasses;
private float[][] outputScores;
private float[] numDetections;
private Map<Integer, Object> outputMap;
/** Input resolution. */
private final int[] inputDims;
/** Final result. */
public OvicDetectionResult result = null;
OvicDetector(InputStream labelInputStream, MappedByteBuffer model) throws IOException {
// Load the label list.
labelList = loadLabelList(labelInputStream);
// Create the TfLite interpreter.
tflite = new Interpreter(model, new Interpreter.Options().setNumThreads(1));
inputDims = TestHelper.getInputDims(tflite, 0);
if (TestHelper.getInputDataType(tflite, 0).equals("float")) {
throw new RuntimeException("The model's input must be QUANTIZED_UINT8.");
}
if (inputDims.length != 4) {
throw new RuntimeException("The model's input dimensions must be 4 (BWHC).");
}
if (inputDims[0] != 1) {
throw new RuntimeException(
"The model must have a batch size of 1, got " + inputDims[0] + " instead.");
}
if (inputDims[3] != 3) {
throw new RuntimeException(
"The model must have three color channels, got " + inputDims[3] + " instead.");
}
// Check the resolution.
int minSide = Math.min(inputDims[1], inputDims[2]);
int maxSide = Math.max(inputDims[1], inputDims[2]);
if (minSide <= 0 || maxSide > 1000) {
throw new RuntimeException("The model's resolution must be between (0, 1000].");
}
// Initialize the input array and result arrays. The input images are stored in a list of
// Object. Since this function anaylzed one image per time, there is only 1 item.
// The output is fomulated as a map of int -> Object. The output arrays are added to the map.
outputLocations = new float[1][NUM_RESULTS][4];
outputClasses = new float[1][NUM_RESULTS];
outputScores = new float[1][NUM_RESULTS];
numDetections = new float[1];
outputMap = new HashMap<>();
outputMap.put(0, outputLocations);
outputMap.put(1, outputClasses);
outputMap.put(2, outputScores);
outputMap.put(3, numDetections);
// Preallocate the result. This will be where inference result is stored after each
// detectByteBuffer call.
result = new OvicDetectionResult(NUM_RESULTS);
}
/** Reads label list from Assets. */
private static List<String> loadLabelList(InputStream labelInputStream) throws IOException {
List<String> labelList = new ArrayList<>();
try (BufferedReader reader =
new BufferedReader(new InputStreamReader(labelInputStream, UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
labelList.add(line);
}
}
return labelList;
}
/**
* The interface to run the detection. This method currently only support float mobilenet_ssd
* model. The quantized models will be added in the future.
*
* @param imgData The image buffer in ByteBuffer format.
* @return boolean indicator of whether detection was a success. If success, the detection results
* is available in the result member variable.
* See OvicDetectionResult.java for details.
*/
boolean detectByteBuffer(ByteBuffer imgData, int imageId) {
if (tflite == null) {
throw new RuntimeException(TAG + ": Detector has not been initialized; Failed.");
}
Object[] inputArray = {imgData};
tflite.runForMultipleInputsOutputs(inputArray, outputMap);
Long latencyMilli = getLastNativeInferenceLatencyMilliseconds();
Long latencyNano = getLastNativeInferenceLatencyNanoseconds();
// Update the results.
result.resetTo(latencyMilli, latencyNano, imageId);
for (int i = 0; i < NUM_RESULTS; i++) {
// The model returns normalized coordinates [start_y, start_x, end_y, end_x].
// The boxes expect pixel coordinates [x1, y1, x2, y2].
// The height and width of the input are in inputDims[1] and inputDims[2].
// The following command converts between model outputs to bounding boxes.
result.addBox(
outputLocations[0][i][1] * inputDims[2],
outputLocations[0][i][0] * inputDims[1],
outputLocations[0][i][3] * inputDims[2],
outputLocations[0][i][2] * inputDims[1],
Math.round(outputClasses[0][i] + 1 /* Label offset */),
outputScores[0][i]);
}
return true; // Marks that the result is available.
}
/**
* Get native inference latency of last image detection run.
*
* @throws RuntimeException if model is uninitialized.
* @return The inference latency in milliseconds.
*/
public Long getLastNativeInferenceLatencyMilliseconds() {
if (tflite == null) {
throw new RuntimeException(TAG + ": ImageNet classifier has not been initialized; Failed.");
}
Long latency = tflite.getLastNativeInferenceDurationNanoseconds();
return (latency == null) ? null : (Long) (latency / 1000000);
}
/**
* Get native inference latency of last image detection run.
*
* @throws RuntimeException if model is uninitialized.
* @return The inference latency in nanoseconds.
*/
public Long getLastNativeInferenceLatencyNanoseconds() {
if (tflite == null) {
throw new IllegalStateException(
TAG + ": ImageNet classifier has not been initialized; Failed.");
}
return tflite.getLastNativeInferenceDurationNanoseconds();
}
public int[] getInputDims() {
return inputDims;
}
public List<String> getLabels() {
return labelList;
}
/** Closes tflite to release resources. */
@Override
public void close() {
tflite.close();
tflite = null;
}
}
@@ -0,0 +1,154 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.ovic;
import android.graphics.Bitmap;
import android.util.Log;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
/**
* Class that benchmarks object detection models.
*/
public final class OvicDetectorBenchmarker extends OvicBenchmarker {
/** Tag for the {@link Log}. */
private static final String TAG = "OvicDetectorBenchmarker";
public double scaleFactorWidth = 1.0f;
public double scaleFactorHeight = 1.0f;
private Bitmap scaledBitmap = null; // Preallocate bitmap for scaling.
private OvicDetector detector;
/**
* Initializes an {@link OvicDetectionBenchmarker}
*
* @param wallTime: a double number specifying the total amount of time to benchmark.
*/
public OvicDetectorBenchmarker(double wallTime) {
super(wallTime);
}
/** Check to see if the detector is ready to test. */
@Override
public boolean readyToTest() {
return (detector != null);
}
/**
* Getting the benchmarker ready for detecting images.
*
* @param labelInputStream: an {@link InputStream} specifying where the list of labels should be
* read from.
* @param model: a {@link MappedByteBuffer} model to benchmark.
*/
@Override
public void getReadyToTest(InputStream labelInputStream, MappedByteBuffer model) {
try {
Log.i(TAG, "Creating detector.");
detector = new OvicDetector(labelInputStream, model);
int[] inputDims = detector.getInputDims();
imgHeight = inputDims[1];
imgWidth = inputDims[2];
imgData = ByteBuffer.allocateDirect(DIM_BATCH_SIZE * imgHeight * imgWidth * DIM_PIXEL_SIZE);
imgData.order(ByteOrder.nativeOrder());
intValues = new int[imgHeight * imgWidth];
benchmarkStarted = false;
} catch (Exception e) {
Log.e(TAG, e.getMessage());
Log.e(TAG, "Failed to initialize COCO detector for the benchmarker.", e);
}
}
/**
* Perform detection on a single ByteBuffer {@link ByteBuffer} image. The image must have the
* same dimension that the model expects.
*
* @param image: a {@link ByteBuffer} image to process.
* @param imageId: an ID uniquely representing the image.
*/
public boolean processBuffer(ByteBuffer image, int imageId) {
if (!readyToTest()) {
return false;
}
try {
if (!detector.detectByteBuffer(image, imageId)) {
return false;
}
} catch (RuntimeException e) {
Log.e(TAG, e.getMessage());
return false;
}
if (!benchmarkStarted) { // Skip the first image to discount warming-up time.
benchmarkStarted = true;
} else {
totalRuntimeNano += ((double) detector.result.latencyNano);
}
return true; // Indicating that result is ready.
}
/**
* Perform detection on a single bitmap image.
*
* @param bitmap: a {@link Bitmap} image to process.
* @param imageId: an ID uniquely representing the image.
*/
@Override
public boolean processBitmap(Bitmap bitmap, int imageId)
throws IOException, InterruptedException {
if (shouldStop() || !readyToTest()) {
return false;
}
convertBitmapToInput(bitmap); // Scale bitmap if needed, store result in imgData.
if (!processBuffer(imgData, imageId)) {
return false;
}
// Scale results back to original image coordinates.
detector.result.scaleUp(scaleFactorWidth, scaleFactorHeight);
return true; // Indicating that result is ready.
}
public OvicDetectionResult getLastDetectionResult() {
return detector.result;
}
@Override
public String getLastResultString() {
if (detector.result == null) {
return null;
}
return detector.result.toString();
}
/**
* Preprocess bitmap image into {@link ByteBuffer} format for the detector.
*
* @param bitmap: a {@link Bitmap} source image.
*/
private void convertBitmapToInput(Bitmap bitmap) {
int originalWidth = bitmap.getWidth();
int originalHeight = bitmap.getHeight();
scaledBitmap = Bitmap.createScaledBitmap(bitmap, imgWidth, imgHeight, true);
scaleFactorWidth = originalWidth * 1.0 / imgWidth;
scaleFactorHeight = originalHeight * 1.0 / imgHeight;
scaledBitmap.getPixels(intValues, 0, imgWidth, 0, 0, imgWidth, imgHeight);
scaledBitmap.recycle();
loadsInputToByteBuffer();
}
}
@@ -0,0 +1,103 @@
/*Copyright 2018 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.ovic;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Random;
/** Validate a submission model. */
public class OvicValidator {
private static void printUsage(PrintStream s) {
s.println("Java program that validates a submission model.");
s.println();
s.println("Usage: ovic_validator <submission file> [<task>]");
s.println();
s.println("Where:");
s.println("<submission file> is the model in TfLite format,");
s.println("<task> is the type of the task: \"classify\" (default) or \"detect\";");
}
public static void main(String[] args) {
if (args.length != 2) {
printUsage(System.err);
System.exit(1);
}
final String modelFile = args[0];
final String taskString = args[1];
final boolean isDetection = taskString.equals("detect");
// Label file for detection is never used, so the same label file is used for both tasks.
final String labelPath =
"tensorflow/lite/java/ovic/src/testdata/labels.txt";
try {
MappedByteBuffer model = loadModelFile(modelFile);
File labelsfile = new File(labelPath);
InputStream labelsInputStream = new FileInputStream(labelsfile);
if (isDetection) {
OvicDetector detector = new OvicDetector(labelsInputStream, model);
int[] inputDims = detector.getInputDims();
ByteBuffer imgData = createByteBuffer(inputDims[1], inputDims[2]);
if (!detector.detectByteBuffer(imgData, /*imageId=*/ 0)) {
throw new RuntimeException("Failed to return detections.");
}
} else {
OvicClassifier classifier = new OvicClassifier(labelsInputStream, model);
int[] inputDims = classifier.getInputDims();
ByteBuffer imgData = createByteBuffer(inputDims[1], inputDims[2]);
OvicClassificationResult testResult = classifier.classifyByteBuffer(imgData);
if (testResult.topKClasses.isEmpty()) {
throw new RuntimeException("Failed to return top K predictions.");
}
}
System.out.printf("Successfully validated %s.%n", modelFile);
} catch (Exception e) {
System.out.println(e.getMessage());
System.out.printf("Failed to validate %s.%n", modelFile);
}
}
private static ByteBuffer createByteBuffer(int imgWidth, int imgHeight) {
ByteBuffer imgData = ByteBuffer.allocateDirect(imgHeight * imgWidth * 3);
imgData.order(ByteOrder.nativeOrder());
Random rand = new Random();
for (int y = 0; y < imgHeight; y++) {
for (int x = 0; x < imgWidth; x++) {
int val = rand.nextInt();
imgData.put((byte) ((val >> 16) & 0xFF));
imgData.put((byte) ((val >> 8) & 0xFF));
imgData.put((byte) (val & 0xFF));
}
}
return imgData;
}
private static MappedByteBuffer loadModelFile(String modelFilePath) throws IOException {
File modelfile = new File(modelFilePath);
FileInputStream inputStream = new FileInputStream(modelfile);
FileChannel fileChannel = inputStream.getChannel();
long startOffset = 0L;
long declaredLength = fileChannel.size();
return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
}
}
@@ -0,0 +1,174 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.ovic;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.fail;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import javax.imageio.ImageIO;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link org.tensorflow.ovic.OvicClassifier}. */
@RunWith(JUnit4.class)
public final class OvicClassifierTest {
private OvicClassifier classifier;
private InputStream labelsInputStream = null;
private MappedByteBuffer quantizedModel = null;
private MappedByteBuffer floatModel = null;
private MappedByteBuffer lowResModel = null;
private ByteBuffer testImage = null;
private ByteBuffer lowResTestImage = null;
private OvicClassificationResult testResult = null;
private static final String LABELS_PATH =
"tensorflow/lite/java/ovic/src/testdata/labels.txt";
private static final String QUANTIZED_MODEL_PATH =
"external/tflite_ovic_testdata/quantized_model.lite";
private static final String LOW_RES_MODEL_PATH =
"external/tflite_ovic_testdata/low_res_model.lite";
private static final String FLOAT_MODEL_PATH =
"external/tflite_ovic_testdata/float_model.lite";
private static final String TEST_IMAGE_PATH =
"external/tflite_ovic_testdata/test_image_224.jpg";
private static final String TEST_LOW_RES_IMAGE_PATH =
"external/tflite_ovic_testdata/test_image_128.jpg";
private static final int TEST_IMAGE_GROUNDTRUTH = 653; // "military uniform"
@Before
public void setUp() {
try {
File labelsfile = new File(LABELS_PATH);
labelsInputStream = new FileInputStream(labelsfile);
quantizedModel = loadModelFile(QUANTIZED_MODEL_PATH);
floatModel = loadModelFile(FLOAT_MODEL_PATH);
lowResModel = loadModelFile(LOW_RES_MODEL_PATH);
File imageFile = new File(TEST_IMAGE_PATH);
BufferedImage img = ImageIO.read(imageFile);
testImage = toByteBuffer(img);
// Low res image and models.
imageFile = new File(TEST_LOW_RES_IMAGE_PATH);
img = ImageIO.read(imageFile);
lowResTestImage = toByteBuffer(img);
} catch (IOException e) {
System.out.print(e.getMessage());
}
System.out.println("Successful setup");
}
@Test
public void ovicClassifier_quantizedModelCreateSuccess() throws Exception {
classifier = new OvicClassifier(labelsInputStream, quantizedModel);
assertThat(classifier).isNotNull();
}
@Test
public void ovicClassifier_floatModelCreateSuccess() throws Exception {
classifier = new OvicClassifier(labelsInputStream, floatModel);
assertThat(classifier).isNotNull();
}
@Test
public void ovicClassifier_quantizedModelClassifySuccess() throws Exception {
classifier = new OvicClassifier(labelsInputStream, quantizedModel);
testResult = classifier.classifyByteBuffer(testImage);
assertCorrectTopK(testResult);
}
@Test
public void ovicClassifier_floatModelClassifySuccess() throws Exception {
classifier = new OvicClassifier(labelsInputStream, floatModel);
testResult = classifier.classifyByteBuffer(testImage);
assertCorrectTopK(testResult);
}
@Test
public void ovicClassifier_lowResModelClassifySuccess() throws Exception {
classifier = new OvicClassifier(labelsInputStream, lowResModel);
testResult = classifier.classifyByteBuffer(lowResTestImage);
assertCorrectTopK(testResult);
}
@Test
public void ovicClassifier_latencyNotNull() throws Exception {
classifier = new OvicClassifier(labelsInputStream, floatModel);
testResult = classifier.classifyByteBuffer(testImage);
assertThat(testResult.latencyNano).isNotNull();
}
@Test
public void ovicClassifier_mismatchedInputResolutionFails() throws Exception {
classifier = new OvicClassifier(labelsInputStream, lowResModel);
int[] inputDims = classifier.getInputDims();
assertThat(inputDims[1]).isEqualTo(128);
assertThat(inputDims[2]).isEqualTo(128);
try {
testResult = classifier.classifyByteBuffer(testImage);
fail();
} catch (IllegalArgumentException e) {
// Success.
}
}
private static ByteBuffer toByteBuffer(BufferedImage image) {
ByteBuffer imgData = ByteBuffer.allocateDirect(
image.getHeight() * image.getWidth() * 3);
imgData.order(ByteOrder.nativeOrder());
for (int y = 0; y < image.getHeight(); y++) {
for (int x = 0; x < image.getWidth(); x++) {
int val = image.getRGB(x, y);
imgData.put((byte) ((val >> 16) & 0xFF));
imgData.put((byte) ((val >> 8) & 0xFF));
imgData.put((byte) (val & 0xFF));
}
}
return imgData;
}
private static void assertCorrectTopK(OvicClassificationResult testResult) {
assertThat(testResult.topKClasses.size()).isGreaterThan(0);
Boolean topKAccurate = false;
// Assert that the correct class is in the top K.
for (int i = 0; i < testResult.topKIndices.size(); i++) {
if (testResult.topKIndices.get(i) == TEST_IMAGE_GROUNDTRUTH) {
topKAccurate = true;
break;
}
}
System.out.println(testResult.toString());
System.out.flush();
assertThat(topKAccurate).isTrue();
}
private static MappedByteBuffer loadModelFile(String modelFilePath) throws IOException {
File modelfile = new File(modelFilePath);
FileInputStream inputStream = new FileInputStream(modelfile);
FileChannel fileChannel = inputStream.getChannel();
long startOffset = 0L;
long declaredLength = fileChannel.size();
return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
}
}
@@ -0,0 +1,131 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.ovic;
import static com.google.common.truth.Truth.assertThat;
import java.awt.Graphics2D;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import javax.imageio.ImageIO;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit test for {@link org.tensorflow.ovic.OvicDetector}. */
@RunWith(JUnit4.class)
public final class OvicDetectorTest {
private OvicDetector detector = null;
private InputStream labelsInputStream = null;
private MappedByteBuffer model = null;
private ByteBuffer testImage = null;
private static final String LABELS_PATH =
"tensorflow/lite/java/ovic/src/testdata/coco_labels.txt";
private static final String MODEL_PATH =
"external/tflite_ovic_testdata/quantized_detect.lite";
private static final String TEST_IMAGE_PATH =
"external/tflite_ovic_testdata/test_image_224.jpg";
private static final int GROUNDTRUTH = 1 /* Person */;
@Before
public void setUp() {
try {
// load models.
model = loadModelFile(MODEL_PATH);
// Load label files;
File labelsfile = new File(LABELS_PATH);
labelsInputStream = new FileInputStream(labelsfile);
// Create detector.
detector = new OvicDetector(labelsInputStream, model);
// Load test image and convert into byte buffer.
File imageFile = new File(TEST_IMAGE_PATH);
BufferedImage rawimg = ImageIO.read(imageFile);
int[] inputDims = detector.getInputDims();
BufferedImage img = new BufferedImage(inputDims[1], inputDims[2], rawimg.getType());
Graphics2D g = img.createGraphics();
g.drawImage(rawimg, 0, 0, inputDims[1], inputDims[2], null);
g.dispose();
testImage = toByteBuffer(img);
} catch (IOException e) {
System.out.println(e.getMessage());
}
System.out.println("Successfully setup");
}
private static MappedByteBuffer loadModelFile(String modelFilePath) throws IOException {
File modelfile = new File(modelFilePath);
FileInputStream inputStream = new FileInputStream(modelfile);
FileChannel fileChannel = inputStream.getChannel();
long startOffset = 0L;
long declaredLength = fileChannel.size();
return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
}
private static ByteBuffer toByteBuffer(BufferedImage image) {
ByteBuffer imgData = ByteBuffer.allocateDirect(image.getHeight() * image.getWidth() * 3);
imgData.order(ByteOrder.nativeOrder());
for (int y = 0; y < image.getHeight(); y++) {
for (int x = 0; x < image.getWidth(); x++) {
int pixelValue = image.getRGB(x, y);
imgData.put((byte) ((pixelValue >> 16) & 0xFF));
imgData.put((byte) ((pixelValue >> 8) & 0xFF));
imgData.put((byte) (pixelValue & 0xFF));
}
}
return imgData;
}
@Test
public void ovicDetector_detectSuccess() throws Exception {
assertThat(detector.detectByteBuffer(testImage, 1)).isTrue();
assertThat(detector.result).isNotNull();
}
@Test
public void ovicDetector_simpleBatchTest() throws Exception {
final int numRepeats = 5;
for (int i = 0; i < numRepeats; i++) {
assertThat(detector.detectByteBuffer(testImage, 1)).isTrue();
OvicDetectionResult result = detector.result;
Boolean detectWithinTop5 = false;
for (int j = 0; j < Math.min(5, result.count); j++) {
if (result.detections.get(j).category == GROUNDTRUTH) {
detectWithinTop5 = true;
break;
}
}
if (!detectWithinTop5) {
System.out.println("---------------- Image " + i + " ---------------------");
System.out.println("Expect category " + GROUNDTRUTH);
System.out.println("Detection results: ");
System.out.println(result.toString());
}
assertThat(detectWithinTop5).isTrue();
}
}
}
+28
View File
@@ -0,0 +1,28 @@
# Testdata for OVIC benchmarker demo App and tests.
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
licenses = ["notice"],
)
filegroup(
name = "ovic_testdata",
srcs = [
"@tflite_ovic_testdata//:detect.lite",
"@tflite_ovic_testdata//:float_model.lite",
"@tflite_ovic_testdata//:low_res_model.lite",
"@tflite_ovic_testdata//:quantized_detect.lite",
"@tflite_ovic_testdata//:quantized_model.lite",
"@tflite_ovic_testdata//:test_image_128.jpg",
"@tflite_ovic_testdata//:test_image_224.jpg",
],
visibility = ["//visibility:public"],
)
exports_files(
[
"labels.txt",
"coco_labels.txt",
],
visibility = ["//visibility:public"],
)
+91
View File
@@ -0,0 +1,91 @@
person
bicycle
car
motorcycle
airplane
bus
train
truck
boat
traffic light
fire hydrant
empty
stop sign
parking meter
bench
bird
cat
dog
horse
sheep
cow
elephant
bear
zebra
giraffe
empty
backpack
umbrella
empty
empty
handbag
tie
suitcase
frisbee
skis
snowboard
sports ball
kite
baseball bat
baseball glove
skateboard
surfboard
tennis racket
bottle
empty
wine glasses
cup
fork
knife
spoon
bowl
banana
apple
sandwich
orange
broccoli
carrot
hot dog
pizza
donut
cake
chair
couch
potted plant
bed
empty
dining table
empty
empty
toilet
empty
tv
laptop
mouse
remote
keyboard
cell phone
microwave
oven
toaster
sink
refrigerator
empty
book
clock
vase
scissors
teddy bear
hair drier
toothbrush
empty
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
-keep class org.tensorflow.lite.annotations.UsedByReflection
-keep @org.tensorflow.lite.annotations.UsedByReflection class *
-keepclassmembers class * {
@org.tensorflow.lite.annotations.UsedByReflection *;
}
@@ -0,0 +1,77 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite;
/** Represents the type of elements in a TensorFlow Lite {@link Tensor} as an enum. */
public enum DataType {
/** 32-bit single precision floating point. */
FLOAT32(1),
/** 32-bit signed integer. */
INT32(2),
/** 8-bit unsigned integer. */
UINT8(3),
/** 64-bit signed integer. */
INT64(4),
/** Strings. */
STRING(5),
/** Bool. */
BOOL(6),
/** 16-bit signed integer. */
INT16(7),
/** 8-bit signed integer. */
INT8(9);
private final int value;
DataType(int value) {
this.value = value;
}
/** Returns the size of an element of this type, in bytes, or -1 if element size is variable. */
public int byteSize() {
switch (this) {
case FLOAT32:
case INT32:
return 4;
case INT16:
return 2;
case INT8:
case UINT8:
return 1;
case INT64:
return 8;
case BOOL:
// Boolean size is JVM-dependent.
return -1;
case STRING:
return -1;
}
throw new IllegalArgumentException(
"DataType error: DataType " + this + " is not supported yet");
}
/** Corresponding value of the TfLiteType enum in the TensorFlow Lite C API. */
int c() {
return value;
}
}
@@ -0,0 +1,72 @@
/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite;
/**
* Utility methods for DataType.
*/
class DataTypeUtils {
private DataTypeUtils() {}
/** Gets string names of the data type. */
static String toStringName(DataType dataType) {
switch (dataType) {
case FLOAT32:
return "float";
case INT32:
return "int";
case INT16:
return "short";
case INT8:
case UINT8:
return "byte";
case INT64:
return "long";
case BOOL:
return "bool";
case STRING:
return "string";
}
throw new IllegalArgumentException(
"DataType error: DataType " + dataType + " is not supported yet");
}
/** Converts a C TfLiteType enum value to the corresponding type. */
static DataType fromC(int c) {
switch (c) {
case 1:
return DataType.FLOAT32;
case 2:
return DataType.INT32;
case 3:
return DataType.UINT8;
case 4:
return DataType.INT64;
case 5:
return DataType.STRING;
case 6:
return DataType.BOOL;
case 7:
return DataType.INT16;
case 9:
return DataType.INT8;
default: // continue below to handle unsupported C types.
}
throw new IllegalArgumentException(
"DataType error: DataType " + c + " is not recognized in Java.");
}
}
@@ -0,0 +1,56 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite;
import java.io.Closeable;
/**
* Wrapper for a native TensorFlow Lite Delegate.
*
* <p>If a delegate implementation holds additional resources or memory that should be explicitly
* freed, then best practice is to add a {@code close()} method to the implementation and have the
* client call that explicitly when the delegate instance is no longer in use. While this approach
* technically allows sharing of a single delegate instance across multiple interpreter instances,
* the delegate implementation must explicitly support this.
*/
public interface Delegate extends Closeable {
/**
* Returns a native handle to the TensorFlow Lite delegate implementation.
*
* <p>Note: The Java {@link Delegate} maintains ownership of the native delegate instance, and
* must ensure its existence for the duration of usage with any {@link InterpreterApi} instance.
*
* <p>Note: the native delegate instance may not be created until the delegate has been attached
* to an interpreter, so this method should not be called until after an interpreter has been
* constructed with this delegate.
*
* @throws IllegalStateException if called before the native delegate instance has been
* constructed.
* @return The native delegate handle. In C/C++, this should be a pointer to
* 'TfLiteOpaqueDelegate'.
*/
long getNativeHandle();
/**
* Closes the delegate and releases any resources associated with it.
*
* <p>In contrast to the method declared in the base {@link Closeable} interface, this method
* does not throw checked exceptions.
*/
@SuppressWarnings("StaticOrDefaultInterfaceMethod")
@Override
default void close() {}
}
@@ -0,0 +1,28 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite;
/** Allows creating delegates for different runtime flavors. */
public interface DelegateFactory {
/**
* Create a {@link Delegate} for the given {@link RuntimeFlavor}.
*
* <p>Note for developers implementing this interface: Currently TF Lite in Google Play Services
* does not support external (developer-provided) delegates. Correspondingly, implementations of
* this method can expect to be called with {@link RuntimeFlavor#APPLICATION}.
*/
Delegate create(RuntimeFlavor runtimeFlavor);
}
@@ -0,0 +1,293 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite;
import java.io.File;
import java.nio.ByteBuffer;
import org.checkerframework.checker.nullness.qual.NonNull;
/**
* Driver class to drive model inference with TensorFlow Lite.
*
* <p>Note: If you don't need access to any of the "experimental" API features below, prefer to use
* InterpreterApi and InterpreterFactory rather than using Interpreter directly.
*
* <p>A {@code Interpreter} encapsulates a pre-trained TensorFlow Lite model, in which operations
* are executed for model inference.
*
* <p>For example, if a model takes only one input and returns only one output:
*
* <pre>{@code
* try (Interpreter interpreter = new Interpreter(file_of_a_tensorflowlite_model)) {
* interpreter.run(input, output);
* }
* }</pre>
*
* <p>If a model takes multiple inputs or outputs:
*
* <pre>{@code
* Object[] inputs = {input0, input1, ...};
* Map<Integer, Object> map_of_indices_to_outputs = new HashMap<>();
* FloatBuffer ith_output = FloatBuffer.allocateDirect(3 * 2 * 4); // Float tensor, shape 3x2x4.
* ith_output.order(ByteOrder.nativeOrder());
* map_of_indices_to_outputs.put(i, ith_output);
* try (Interpreter interpreter = new Interpreter(file_of_a_tensorflowlite_model)) {
* interpreter.runForMultipleInputsOutputs(inputs, map_of_indices_to_outputs);
* }
* }</pre>
*
* <p>If a model takes or produces string tensors:
*
* <pre>{@code
* String[] input = {"foo", "bar"}; // Input tensor shape is [2].
* String[][] output = new String[3][2]; // Output tensor shape is [3, 2].
* try (Interpreter interpreter = new Interpreter(file_of_a_tensorflowlite_model)) {
* interpreter.runForMultipleInputsOutputs(input, output);
* }
* }</pre>
*
* <p>Note that there's a distinction between shape [] and shape[1]. For scalar string tensor
* outputs:
*
* <pre>{@code
* String[] input = {"foo"}; // Input tensor shape is [1].
* ByteBuffer outputBuffer = ByteBuffer.allocate(OUTPUT_BYTES_SIZE); // Output tensor shape is [].
* try (Interpreter interpreter = new Interpreter(file_of_a_tensorflowlite_model)) {
* interpreter.runForMultipleInputsOutputs(input, outputBuffer);
* }
* byte[] outputBytes = new byte[outputBuffer.remaining()];
* outputBuffer.get(outputBytes);
* // Below, the `charset` can be StandardCharsets.UTF_8.
* String output = new String(outputBytes, charset);
* }</pre>
*
* <p>Orders of inputs and outputs are determined when converting TensorFlow model to TensorFlowLite
* model with Toco, as are the default shapes of the inputs.
*
* <p>When inputs are provided as (multi-dimensional) arrays, the corresponding input tensor(s) will
* be implicitly resized according to that array's shape. When inputs are provided as {@code Buffer}
* types, no implicit resizing is done; the caller must ensure that the {@code Buffer} byte size
* either matches that of the corresponding tensor, or that they first resize the tensor via {@link
* #resizeInput(int, int[])}. Tensor shape and type information can be obtained via the {@link
* Tensor} class, available via {@link #getInputTensor(int)} and {@link #getOutputTensor(int)}.
*
* <p><b>WARNING:</b>{@code Interpreter} instances are <b>not</b> thread-safe. A {@code Interpreter}
* owns resources that <b>must</b> be explicitly freed by invoking {@link #close()}
*
* <p>The TFLite library is built against NDK API 19. It may work for Android API levels below 19,
* but is not guaranteed.
*/
public final class Interpreter extends InterpreterImpl implements InterpreterApi {
/** An options class for controlling runtime interpreter behavior. */
public static class Options extends InterpreterImpl.Options {
public Options() {}
public Options(InterpreterApi.Options options) {
super(options);
}
Options(InterpreterImpl.Options options) {
super(options);
}
@Override
public Options setUseXNNPACK(boolean useXNNPACK) {
super.setUseXNNPACK(useXNNPACK);
return this;
}
@Override
public Options setNumThreads(int numThreads) {
super.setNumThreads(numThreads);
return this;
}
@Override
public Options setUseNNAPI(boolean useNNAPI) {
super.setUseNNAPI(useNNAPI);
return this;
}
/**
* Sets whether to allow float16 precision for FP32 calculation when possible. Defaults to false
* (disallow).
*
* @deprecated Prefer using <a
* href="https://github.com/tensorflow/tensorflow/blob/5dc7f6981fdaf74c8c5be41f393df705841fb7c5/tensorflow/lite/delegates/nnapi/java/src/main/java/org/tensorflow/lite/nnapi/NnApiDelegate.java#L127">NnApiDelegate.Options#setAllowFp16(boolean
* enable)</a>.
*/
@Deprecated
public Options setAllowFp16PrecisionForFp32(boolean allow) {
this.allowFp16PrecisionForFp32 = allow;
return this;
}
@Override
public Options addDelegate(Delegate delegate) {
super.addDelegate(delegate);
return this;
}
@Override
public Options addDelegateFactory(DelegateFactory delegateFactory) {
super.addDelegateFactory(delegateFactory);
return this;
}
/**
* Advanced: Set if buffer handle output is allowed.
*
* <p>When a {@link Delegate} supports hardware acceleration, the interpreter will make the data
* of output tensors available in the CPU-allocated tensor buffers by default. If the client can
* consume the buffer handle directly (e.g. reading output from OpenGL texture), it can set this
* flag to false, avoiding the copy of data to the CPU buffer. The delegate documentation should
* indicate whether this is supported and how it can be used.
*
* <p>WARNING: This is an experimental interface that is subject to change.
*/
public Options setAllowBufferHandleOutput(boolean allow) {
this.allowBufferHandleOutput = allow;
return this;
}
/**
* Enables or disables compression of identical per-channel quantization zero-points into a
* single value to reduce memory usage.
*
* <p>value whether to enable compression of identical per-channel quantization zero-points.
*
* <p>WARNING: This is an experimental interface that is subject to change.
*/
public Options setCompressQuantizationZeroPoints(boolean value) {
this.compressQuantizationZeroPoints = value;
return this;
}
/**
* Returns whether compression of identical per-channel quantization zero-points is enabled.
*
* <p>WARNING: This is an experimental interface that is subject to change.
*/
public boolean getCompressQuantizationZeroPoints() {
return compressQuantizationZeroPoints != null && compressQuantizationZeroPoints;
}
@Override
public Options setCancellable(boolean allow) {
super.setCancellable(allow);
return this;
}
@Override
public Options setRuntime(InterpreterApi.Options.TfLiteRuntime runtime) {
super.setRuntime(runtime);
return this;
}
}
/**
* Initializes an {@code Interpreter}.
*
* @param modelFile a File of a pre-trained TF Lite model.
* @throws IllegalArgumentException if {@code modelFile} does not encode a valid TensorFlow Lite
* model.
*/
public Interpreter(@NonNull File modelFile) {
this(modelFile, /* options= */ null);
}
/**
* Initializes an {@code Interpreter} and specifies options for customizing interpreter behavior.
*
* @param modelFile a file of a pre-trained TF Lite model
* @param options a set of options for customizing interpreter behavior
* @throws IllegalArgumentException if {@code modelFile} does not encode a valid TensorFlow Lite
* model.
*/
public Interpreter(@NonNull File modelFile, Options options) {
this(new NativeInterpreterWrapperExperimental(modelFile.getAbsolutePath(), options));
}
/**
* Initializes an {@code Interpreter} with a {@code ByteBuffer} of a model file.
*
* <p>The ByteBuffer should not be modified after the construction of a {@code Interpreter}. The
* {@code ByteBuffer} can be either a {@code MappedByteBuffer} that memory-maps a model file, or a
* direct {@code ByteBuffer} of nativeOrder() that contains the bytes content of a model.
*
* @throws IllegalArgumentException if {@code byteBuffer} is not a {@code MappedByteBuffer} nor a
* direct {@code ByteBuffer} of nativeOrder.
*/
public Interpreter(@NonNull ByteBuffer byteBuffer) {
this(byteBuffer, /* options= */ null);
}
/**
* Initializes an {@code Interpreter} with a {@code ByteBuffer} of a model file and a set of
* custom {@link Interpreter.Options}.
*
* <p>The {@code ByteBuffer} should not be modified after the construction of an {@code
* Interpreter}. The {@code ByteBuffer} can be either a {@code MappedByteBuffer} that memory-maps
* a model file, or a direct {@code ByteBuffer} of nativeOrder() that contains the bytes content
* of a model.
*
* @throws IllegalArgumentException if {@code byteBuffer} is not a {@code MappedByteBuffer} nor a
* direct {@code ByteBuffer} of nativeOrder.
*/
public Interpreter(@NonNull ByteBuffer byteBuffer, Options options) {
this(new NativeInterpreterWrapperExperimental(byteBuffer, options));
}
private Interpreter(NativeInterpreterWrapperExperimental wrapper) {
super(wrapper);
wrapperExperimental = wrapper;
}
/**
* Advanced: Resets all variable tensors to the default value.
*
* <p>If a variable tensor doesn't have an associated buffer, it will be reset to zero.
*
* <p>WARNING: This is an experimental API and subject to change.
*/
public void resetVariableTensors() {
checkNotClosed();
wrapperExperimental.resetVariableTensors();
}
/**
* Advanced: Interrupts inference in the middle of a call to {@link Interpreter#run}.
*
* <p>A cancellation flag will be set to true when this function gets called. The interpreter will
* check the flag between Op invocations, and if it's {@code true}, the interpreter will stop
* execution. The interpreter will remain a cancelled state until explicitly "uncancelled" by
* {@code setCancelled(false)}.
*
* <p>WARNING: This is an experimental API and subject to change.
*
* @param cancelled {@code true} to cancel inference in a best-effort way; {@code false} to
* resume.
* @throws IllegalStateException if the interpreter is not initialized with the cancellable
* option, which is by default off.
* @see Interpreter.Options#setCancellable(boolean).
*/
public void setCancelled(boolean cancelled) {
wrapper.setCancelled(cancelled);
}
private final NativeInterpreterWrapperExperimental wrapperExperimental;
}
@@ -0,0 +1,625 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite;
import java.io.File;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.tensorflow.lite.InterpreterApi.Options.TfLiteRuntime;
import org.tensorflow.lite.acceleration.ValidatedAccelerationConfig;
import org.tensorflow.lite.nnapi.NnApiDelegate;
/**
* Interface to TensorFlow Lite model interpreter, excluding experimental methods.
*
* <p>An {@code InterpreterApi} instance encapsulates a pre-trained TensorFlow Lite model, in which
* operations are executed for model inference.
*
* <p>For example, if a model takes only one input and returns only one output:
*
* <pre>{@code
* try (InterpreterApi interpreter =
* new InterpreterApi.create(file_of_a_tensorflowlite_model)) {
* interpreter.run(input, output);
* }
* }</pre>
*
* <p>If a model takes multiple inputs or outputs:
*
* <pre>{@code
* Object[] inputs = {input0, input1, ...};
* Map<Integer, Object> map_of_indices_to_outputs = new HashMap<>();
* FloatBuffer ith_output = FloatBuffer.allocateDirect(3 * 2 * 4); // Float tensor, shape 3x2x4.
* ith_output.order(ByteOrder.nativeOrder());
* map_of_indices_to_outputs.put(i, ith_output);
* try (InterpreterApi interpreter =
* new InterpreterApi.create(file_of_a_tensorflowlite_model)) {
* interpreter.runForMultipleInputsOutputs(inputs, map_of_indices_to_outputs);
* }
* }</pre>
*
* <p>If a model takes or produces string tensors:
*
* <pre>{@code
* String[] input = {"foo", "bar"}; // Input tensor shape is [2].
* String[][] output = new String[3][2]; // Output tensor shape is [3, 2].
* try (InterpreterApi interpreter =
* new InterpreterApi.create(file_of_a_tensorflowlite_model)) {
* interpreter.runForMultipleInputsOutputs(input, output);
* }
* }</pre>
*
* <p>Note that there's a distinction between shape [] and shape[1]. For scalar string tensor
* outputs:
*
* <pre>{@code
* String[] input = {"foo"}; // Input tensor shape is [1].
* ByteBuffer outputBuffer = ByteBuffer.allocate(OUTPUT_BYTES_SIZE); // Output tensor shape is [].
* try (Interpreter interpreter = new Interpreter(file_of_a_tensorflowlite_model)) {
* interpreter.runForMultipleInputsOutputs(input, outputBuffer);
* }
* byte[] outputBytes = new byte[outputBuffer.remaining()];
* outputBuffer.get(outputBytes);
* // Below, the `charset` can be StandardCharsets.UTF_8.
* String output = new String(outputBytes, charset);
* }</pre>
*
* <p>Orders of inputs and outputs are determined when converting TensorFlow model to TensorFlowLite
* model with Toco, as are the default shapes of the inputs.
*
* <p>When inputs are provided as (multi-dimensional) arrays, the corresponding input tensor(s) will
* be implicitly resized according to that array's shape. When inputs are provided as {@link
* java.nio.Buffer} types, no implicit resizing is done; the caller must ensure that the {@link
* java.nio.Buffer} byte size either matches that of the corresponding tensor, or that they first
* resize the tensor via {@link #resizeInput(int, int[])}. Tensor shape and type information can be
* obtained via the {@link Tensor} class, available via {@link #getInputTensor(int)} and {@link
* #getOutputTensor(int)}.
*
* <p><b>WARNING:</b>{@code InterpreterApi} instances are <b>not</b> thread-safe.
*
* <p><b>WARNING:</b>An {@code InterpreterApi} instance owns resources that <b>must</b> be
* explicitly freed by invoking {@link #close()}
*
* <p>The TFLite library is built against NDK API 19. It may work for Android API levels below 19,
* but is not guaranteed.
*/
public interface InterpreterApi extends AutoCloseable {
/** An options class for controlling runtime interpreter behavior. */
class Options {
public Options() {
this.delegates = new ArrayList<>();
this.delegateFactories = new ArrayList<>();
}
public Options(Options other) {
this.numThreads = other.numThreads;
this.useNNAPI = other.useNNAPI;
this.allowCancellation = other.allowCancellation;
this.delegates = new ArrayList<>(other.delegates);
this.delegateFactories = new ArrayList<>(other.delegateFactories);
this.runtime = other.runtime;
this.validatedAccelerationConfig = other.validatedAccelerationConfig;
this.useXNNPACK = other.useXNNPACK;
}
/**
* Sets the number of threads to be used for ops that support multi-threading.
*
* <p>{@code numThreads} should be {@code >= -1}. Setting {@code numThreads} to 0 has the effect
* of disabling multithreading, which is equivalent to setting {@code numThreads} to 1. If
* unspecified, or set to the value -1, the number of threads used will be
* implementation-defined and platform-dependent.
*/
public Options setNumThreads(int numThreads) {
this.numThreads = numThreads;
return this;
}
/**
* Returns the number of threads to be used for ops that support multi-threading.
*
* <p>{@code numThreads} should be {@code >= -1}. Values of 0 (or 1) disable multithreading.
* Default value is -1: the number of threads used will be implementation-defined and
* platform-dependent.
*/
public int getNumThreads() {
return numThreads;
}
/** Sets whether to use NN API (if available) for op execution. Defaults to false (disabled). */
public Options setUseNNAPI(boolean useNNAPI) {
this.useNNAPI = useNNAPI;
return this;
}
/**
* Returns whether to use NN API (if available) for op execution. Default value is false
* (disabled).
*/
public boolean getUseNNAPI() {
return useNNAPI != null && useNNAPI;
}
/**
* Advanced: Set if the interpreter is able to be cancelled.
*
* <p>Interpreters may have an experimental API <a
* href="https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/Interpreter#setCancelled(boolean)">setCancelled(boolean)</a>.
* If this interpreter is cancellable and such a method is invoked, a cancellation flag will be
* set to true. The interpreter will check the flag between Op invocations, and if it's {@code
* true}, the interpreter will stop execution. The interpreter will remain a cancelled state
* until explicitly "uncancelled" by {@code setCancelled(false)}.
*/
public Options setCancellable(boolean allow) {
this.allowCancellation = allow;
return this;
}
/**
* Advanced: Returns whether the interpreter is able to be cancelled.
*
* <p>Interpreters may have an experimental API <a
* href="https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/Interpreter#setCancelled(boolean)">setCancelled(boolean)</a>.
* If this interpreter is cancellable and such a method is invoked, a cancellation flag will be
* set to true. The interpreter will check the flag between Op invocations, and if it's {@code
* true}, the interpreter will stop execution. The interpreter will remain a cancelled state
* until explicitly "uncancelled" by {@code setCancelled(false)}.
*/
public boolean isCancellable() {
return allowCancellation != null && allowCancellation;
}
/**
* Adds a {@link Delegate} to be applied during interpreter creation.
*
* <p>Delegates added here are applied before any delegates created from a {@link
* DelegateFactory} that was added with {@link #addDelegateFactory}.
*
* <p>Note that TF Lite in Google Play Services (see {@link #setRuntime}) does not support
* external (developer-provided) delegates, and adding a {@link Delegate} other than {@link
* NnApiDelegate} here is not allowed when using TF Lite in Google Play Services.
*/
public Options addDelegate(Delegate delegate) {
delegates.add(delegate);
return this;
}
/**
* Returns the list of delegates intended to be applied during interpreter creation that have
* been registered via {@code addDelegate}.
*/
public List<Delegate> getDelegates() {
return Collections.unmodifiableList(delegates);
}
/**
* Adds a {@link DelegateFactory} which will be invoked to apply its created {@link Delegate}
* during interpreter creation.
*
* <p>Delegates from a delegated factory that was added here are applied after any delegates
* added with {@link #addDelegate}.
*/
public Options addDelegateFactory(DelegateFactory delegateFactory) {
delegateFactories.add(delegateFactory);
return this;
}
/**
* Returns the list of delegate factories that have been registered via {@code
* addDelegateFactory}).
*/
public List<DelegateFactory> getDelegateFactories() {
return Collections.unmodifiableList(delegateFactories);
}
/**
* Enum to represent where to get the TensorFlow Lite runtime implementation from.
*
* <p>The difference between this class and the RuntimeFlavor class: This class specifies a
* <em>preference</em> which runtime to use, whereas {@link RuntimeFlavor} specifies which exact
* runtime <em>is</em> being used.
*/
public enum TfLiteRuntime {
/**
* Use a TF Lite runtime implementation that is linked into the application. If there is no
* suitable TF Lite runtime implementation linked into the application, then attempting to
* create an InterpreterApi instance with this TfLiteRuntime setting will throw an
* IllegalStateException exception (even if the OS or system services could provide a TF Lite
* runtime implementation).
*
* <p>This is the default setting. This setting is also appropriate for apps that must run on
* systems that don't provide a TF Lite runtime implementation.
*/
FROM_APPLICATION_ONLY,
/**
* Use a TF Lite runtime implementation provided by the OS or system services. This will be
* obtained from a system library / shared object / service, such as Google Play Services. It
* may be newer than the version linked into the application (if any). If there is no suitable
* TF Lite runtime implementation provided by the system, then attempting to create an
* InterpreterApi instance with this TfLiteRuntime setting will throw an IllegalStateException
* exception (even if there is a TF Lite runtime implementation linked into the application).
*
* <p>This setting is appropriate for code that will use a system-provided TF Lite runtime,
* which can reduce app binary size and can be updated more frequently.
*/
FROM_SYSTEM_ONLY,
/**
* Use a system-provided TF Lite runtime implementation, if any, otherwise use the TF Lite
* runtime implementation linked into the application, if any. If no suitable TF Lite runtime
* can be found in any location, then attempting to create an InterpreterApi instance with
* this TFLiteRuntime setting will throw an IllegalStateException. If there is both a suitable
* TF Lite runtime linked into the application and also a suitable TF Lite runtime provided by
* the system, the one provided by the system will be used.
*
* <p>This setting is suitable for use in code that doesn't care where the TF Lite runtime is
* coming from (e.g. middleware layers).
*/
PREFER_SYSTEM_OVER_APPLICATION,
}
/** Specify where to get the TF Lite runtime implementation from. */
public Options setRuntime(TfLiteRuntime runtime) {
this.runtime = runtime;
return this;
}
/** Return where to get the TF Lite runtime implementation from. */
public TfLiteRuntime getRuntime() {
return runtime;
}
/** Specify the acceleration configuration. */
public Options setAccelerationConfig(ValidatedAccelerationConfig config) {
this.validatedAccelerationConfig = config;
return this;
}
/** Return the acceleration configuration. */
public ValidatedAccelerationConfig getAccelerationConfig() {
return this.validatedAccelerationConfig;
}
/**
* Enable or disable an optimized set of CPU kernels (provided by XNNPACK). Enabled by default.
*/
public Options setUseXNNPACK(boolean useXNNPACK) {
this.useXNNPACK = useXNNPACK;
return this;
}
public boolean getUseXNNPACK() {
// A null value indicates the default behavior, which is currently to apply the delegate.
return useXNNPACK == null || useXNNPACK.booleanValue();
}
TfLiteRuntime runtime = TfLiteRuntime.FROM_APPLICATION_ONLY;
int numThreads = -1;
Boolean useNNAPI;
/**
* Note: the initial "null" value indicates default behavior (XNNPACK delegate will be applied
* by default whenever possible).
*
* <p>Disabling this flag will disable use of a highly optimized set of CPU kernels provided via
* the XNNPACK delegate. Currently, this is restricted to a subset of floating point operations.
* See
* https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/delegates/xnnpack/README.md
* for more details.
*/
Boolean useXNNPACK;
Boolean allowCancellation;
ValidatedAccelerationConfig validatedAccelerationConfig;
// See InterpreterApi.Options#addDelegate.
final List<Delegate> delegates;
// See InterpreterApi.Options#addDelegateFactory.
private final List<DelegateFactory> delegateFactories;
}
/**
* Constructs an {@link InterpreterApi} instance, using the specified model and options. The model
* will be loaded from a file.
*
* @param modelFile A file containing a pre-trained TF Lite model.
* @param options A set of options for customizing interpreter behavior.
* @throws IllegalArgumentException if {@code modelFile} does not encode a valid TensorFlow Lite
* model.
*/
@SuppressWarnings("StaticOrDefaultInterfaceMethod")
static InterpreterApi create(@NonNull File modelFile, InterpreterApi.Options options) {
TfLiteRuntime runtime = (options == null ? null : options.getRuntime());
InterpreterFactoryApi factory = TensorFlowLite.getFactory(runtime);
return factory.create(modelFile, options);
}
/**
* Constructs an {@link InterpreterApi} instance, using the specified model and options. The model
* will be read from a {@code ByteBuffer}.
*
* @param byteBuffer A pre-trained TF Lite model, in binary serialized form. The ByteBuffer should
* not be modified after the construction of an {@link InterpreterApi} instance. The {@code
* ByteBuffer} can be either a {@code MappedByteBuffer} that memory-maps a model file, or a
* direct {@code ByteBuffer} of nativeOrder() that contains the bytes content of a model.
* @param options A set of options for customizing interpreter behavior.
* @throws IllegalArgumentException if {@code byteBuffer} is not a {@code MappedByteBuffer} nor a
* direct {@code ByteBuffer} of nativeOrder.
*/
@SuppressWarnings("StaticOrDefaultInterfaceMethod")
static InterpreterApi create(@NonNull ByteBuffer byteBuffer, InterpreterApi.Options options) {
TfLiteRuntime runtime = (options == null ? null : options.getRuntime());
InterpreterFactoryApi factory = TensorFlowLite.getFactory(runtime);
return factory.create(byteBuffer, options);
}
/**
* Runs model inference if the model takes only one input, and provides only one output.
*
* <p>Warning: The API is more efficient if a {@code Buffer} (preferably direct, but not required)
* is used as the input/output data type. Please consider using {@code Buffer} to feed and fetch
* primitive data for better performance. The following concrete {@code Buffer} types are
* supported:
*
* <ul>
* <li>{@code ByteBuffer} - compatible with any underlying primitive Tensor type.
* <li>{@code FloatBuffer} - compatible with float Tensors.
* <li>{@code IntBuffer} - compatible with int32 Tensors.
* <li>{@code LongBuffer} - compatible with int64 Tensors.
* </ul>
*
* Note that boolean types are only supported as arrays, not {@code Buffer}s, or as scalar inputs.
*
* @param input an array or multidimensional array, or a {@code Buffer} of primitive types
* including int, float, long, and byte. {@code Buffer} is the preferred way to pass large
* input data for primitive types, whereas string types require using the (multi-dimensional)
* array input path. When a {@code Buffer} is used, its content should remain unchanged until
* model inference is done, and the caller must ensure that the {@code Buffer} is at the
* appropriate read position. A {@code null} value is allowed only if the caller is using a
* {@link Delegate} that allows buffer handle interop, and such a buffer has been bound to the
* input {@link Tensor}.
* @param output a multidimensional array of output data, or a {@code Buffer} of primitive types
* including int, float, long, and byte. When a {@code Buffer} is used, the caller must ensure
* that it is set the appropriate write position. A null value is allowed, and is useful for
* certain cases, e.g., if the caller is using a {@link Delegate} that allows buffer handle
* interop, and such a buffer has been bound to the output {@link Tensor} (see also <a
* href="https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/Interpreter.Options#setAllowBufferHandleOutput(boolean)">Interpreter.Options#setAllowBufferHandleOutput(boolean)</a>),
* or if the graph has dynamically shaped outputs and the caller must query the output {@link
* Tensor} shape after inference has been invoked, fetching the data directly from the output
* tensor (via {@link Tensor#asReadOnlyBuffer()}).
* @throws IllegalArgumentException if {@code input} is null or empty, or if an error occurs when
* running inference.
* @throws IllegalArgumentException (EXPERIMENTAL, subject to change) if the inference is
* interrupted by {@code setCancelled(true)}.
*/
void run(Object input, Object output);
/**
* Runs model inference if the model takes multiple inputs, or returns multiple outputs.
*
* <p>Warning: The API is more efficient if {@code Buffer}s (preferably direct, but not required)
* are used as the input/output data types. Please consider using {@code Buffer} to feed and fetch
* primitive data for better performance. The following concrete {@code Buffer} types are
* supported:
*
* <ul>
* <li>{@code ByteBuffer} - compatible with any underlying primitive Tensor type.
* <li>{@code FloatBuffer} - compatible with float Tensors.
* <li>{@code IntBuffer} - compatible with int32 Tensors.
* <li>{@code LongBuffer} - compatible with int64 Tensors.
* </ul>
*
* Note that boolean types are only supported as arrays, not {@code Buffer}s, or as scalar inputs.
*
* <p>Note: {@code null} values for individual elements of {@code inputs} and {@code outputs} is
* allowed only if the caller is using a {@link Delegate} that allows buffer handle interop, and
* such a buffer has been bound to the corresponding input or output {@link Tensor}(s).
*
* @param inputs an array of input data. The inputs should be in the same order as inputs of the
* model. Each input can be an array or multidimensional array, or a {@code Buffer} of
* primitive types including int, float, long, and byte. {@code Buffer} is the preferred way
* to pass large input data, whereas string types require using the (multi-dimensional) array
* input path. When {@code Buffer} is used, its content should remain unchanged until model
* inference is done, and the caller must ensure that the {@code Buffer} is at the appropriate
* read position.
* @param outputs a map mapping output indices to multidimensional arrays of output data or {@code
* Buffer}s of primitive types including int, float, long, and byte. It only needs to keep
* entries for the outputs to be used. When a {@code Buffer} is used, the caller must ensure
* that it is set the appropriate write position. The map may be empty for cases where either
* buffer handles are used for output tensor data, or cases where the outputs are dynamically
* shaped and the caller must query the output {@link Tensor} shape after inference has been
* invoked, fetching the data directly from the output tensor (via {@link
* Tensor#asReadOnlyBuffer()}).
* @throws IllegalArgumentException if {@code inputs} is null or empty, if {@code outputs} is
* null, or if an error occurs when running inference.
*/
void runForMultipleInputsOutputs(
Object @NonNull [] inputs, @NonNull Map<Integer, Object> outputs);
/**
* Explicitly updates allocations for all tensors, if necessary.
*
* <p>This will propagate shapes and memory allocations for dependent tensors using the input
* tensor shape(s) as given.
*
* <p>Note: This call is *purely optional*. Tensor allocation will occur automatically during
* execution if any input tensors have been resized. This call is most useful in determining the
* shapes for any output tensors before executing the graph, e.g.,
*
* <pre> {@code
* interpreter.resizeInput(0, new int[]{1, 4, 4, 3}));
* interpreter.allocateTensors();
* FloatBuffer input = FloatBuffer.allocate(interpreter.getInputTensor(0).numElements());
* // Populate inputs...
* FloatBuffer output = FloatBuffer.allocate(interpreter.getOutputTensor(0).numElements());
* interpreter.run(input, output)
* // Process outputs...}</pre>
*
* <p>Note: Some graphs have dynamically shaped outputs, in which case the output shape may not
* fully propagate until inference is executed.
*
* @throws IllegalStateException if the graph's tensors could not be successfully allocated.
*/
void allocateTensors();
/**
* Resizes idx-th input of the native model to the given dims.
*
* @throws IllegalArgumentException if {@code idx} is negative or is not smaller than the number
* of model inputs; or if error occurs when resizing the idx-th input.
*/
void resizeInput(int idx, int @NonNull [] dims);
/**
* Resizes idx-th input of the native model to the given dims.
*
* <p>When `strict` is True, only unknown dimensions can be resized. Unknown dimensions are
* indicated as `-1` in the array returned by `Tensor.shapeSignature()`.
*
* @throws IllegalArgumentException if {@code idx} is negative or is not smaller than the number
* of model inputs; or if error occurs when resizing the idx-th input. Additionally, the error
* occurs when attempting to resize a tensor with fixed dimensions when `strict` is True.
*/
void resizeInput(int idx, int @NonNull [] dims, boolean strict);
/** Gets the number of input tensors. */
int getInputTensorCount();
/**
* Gets index of an input given the op name of the input.
*
* @throws IllegalArgumentException if {@code opName} does not match any input in the model used
* to initialize the interpreter.
*/
int getInputIndex(String opName);
/**
* Gets the Tensor associated with the provided input index.
*
* @throws IllegalArgumentException if {@code inputIndex} is negative or is not smaller than the
* number of model inputs.
*/
Tensor getInputTensor(int inputIndex);
/** Gets the number of output Tensors. */
int getOutputTensorCount();
/**
* Gets index of an output given the op name of the output.
*
* @throws IllegalArgumentException if {@code opName} does not match any output in the model used
* to initialize the interpreter.
*/
int getOutputIndex(String opName);
/**
* Gets the Tensor associated with the provided output index.
*
* <p>Note: Output tensor details (e.g., shape) may not be fully populated until after inference
* is executed. If you need updated details *before* running inference (e.g., after resizing an
* input tensor, which may invalidate output tensor shapes), use {@link #allocateTensors()} to
* explicitly trigger allocation and shape propagation. Note that, for graphs with output shapes
* that are dependent on input *values*, the output shape may not be fully determined until
* running inference.
*
* @throws IllegalArgumentException if {@code outputIndex} is negative or is not smaller than the
* number of model outputs.
*/
Tensor getOutputTensor(int outputIndex);
/**
* Runs model inference based on SignatureDef provided through {@code signatureKey}.
*
* <p>See {@link Interpreter#run(Object, Object)} for more details on the allowed input and output
* data types.
*
* @param inputs A map from input name in the SignatureDef to an input object.
* @param outputs A map from output name in SignatureDef to output data. This may be empty if the
* caller wishes to query the {@link Tensor} data directly after inference (e.g., if the
* output shape is dynamic, or output buffer handles are used).
* @param signatureKey Signature key identifying the SignatureDef.
* @throws IllegalArgumentException if {@code inputs} is null or empty, if {@code outputs} or
* {@code signatureKey} is null, or if an error occurs when running inference.
*/
public void runSignature(
@NonNull Map<String, Object> inputs,
@NonNull Map<String, Object> outputs,
String signatureKey);
/**
* Same as {@link #runSignature(Map, Map, String)} but doesn't require passing a signatureKey,
* assuming the model has one SignatureDef. If the model has more than one SignatureDef it will
* throw an exception.
*/
public void runSignature(
@NonNull Map<String, Object> inputs, @NonNull Map<String, Object> outputs);
/**
* Gets the Tensor associated with the provided input name and signature method name.
*
* @param inputName Input name in the signature.
* @param signatureKey Signature key identifying the SignatureDef, can be null if the model has
* one signature.
* @throws IllegalArgumentException if {@code inputName} or {@code signatureKey} is null or empty,
* or invalid name provided.
*/
public Tensor getInputTensorFromSignature(String inputName, String signatureKey);
/** Gets the list of SignatureDef exported method names available in the model. */
public String[] getSignatureKeys();
/** Gets the list of SignatureDefs inputs for method {@code signatureKey}. */
public String[] getSignatureInputs(String signatureKey);
/** Gets the list of SignatureDefs outputs for method {@code signatureKey}. */
public String[] getSignatureOutputs(String signatureKey);
/**
* Gets the Tensor associated with the provided output name in specific signature method.
*
* <p>Note: Output tensor details (e.g., shape) may not be fully populated until after inference
* is executed. If you need updated details *before* running inference (e.g., after resizing an
* input tensor, which may invalidate output tensor shapes), use {@link #allocateTensors()} to
* explicitly trigger allocation and shape propagation. Note that, for graphs with output shapes
* that are dependent on input *values*, the output shape may not be fully determined until
* running inference.
*
* @param outputName Output name in the signature.
* @param signatureKey Signature key identifying the SignatureDef, can be null if the model has
* one signature.
* @throws IllegalArgumentException if {@code outputName} or {@code signatureKey} is null or
* empty, or invalid name provided.
*/
public Tensor getOutputTensorFromSignature(String outputName, String signatureKey);
/**
* Returns native inference timing.
*
* @throws IllegalArgumentException if the model is not initialized by the interpreter.
*/
Long getLastNativeInferenceDurationNanoseconds();
/** Release resources associated with the {@code InterpreterApi} instance. */
@Override
void close();
}
@@ -0,0 +1,59 @@
/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite;
import java.io.File;
import java.nio.ByteBuffer;
import org.checkerframework.checker.nullness.qual.NonNull;
/**
* Factory for constructing InterpreterApi instances.
*
* <p>Deprecated; please use the InterpreterApi.create method instead.
*/
@Deprecated
public class InterpreterFactory {
public InterpreterFactory() {}
/**
* Constructs an {@link InterpreterApi} instance, using the specified model and options. The model
* will be loaded from a file.
*
* @param modelFile A file containing a pre-trained TF Lite model.
* @param options A set of options for customizing interpreter behavior.
* @throws IllegalArgumentException if {@code modelFile} does not encode a valid TensorFlow Lite
* model.
*/
public InterpreterApi create(@NonNull File modelFile, InterpreterApi.Options options) {
return InterpreterApi.create(modelFile, options);
}
/**
* Constructs an {@link InterpreterApi} instance, using the specified model and options. The model
* will be read from a {@code ByteBuffer}.
*
* @param byteBuffer A pre-trained TF Lite model, in binary serialized form. The ByteBuffer should
* not be modified after the construction of an {@link InterpreterApi} instance. The {@code
* ByteBuffer} can be either a {@code MappedByteBuffer} that memory-maps a model file, or a
* direct {@code ByteBuffer} of nativeOrder() that contains the bytes content of a model.
* @param options A set of options for customizing interpreter behavior.
* @throws IllegalArgumentException if {@code byteBuffer} is not a {@code MappedByteBuffer} nor a
* direct {@code ByteBuffer} of nativeOrder.
*/
public InterpreterApi create(@NonNull ByteBuffer byteBuffer, InterpreterApi.Options options) {
return InterpreterApi.create(byteBuffer, options);
}
}

Some files were not shown because too many files have changed in this diff Show More