wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions 3 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/java/BUILD
+479
View File
@@ -0,0 +1,479 @@
# Description:
# TensorFlow Java API.
load("@rules_cc//cc:cc_library.bzl", "cc_library")
load("@rules_java//java:defs.bzl", "java_library", "java_plugin")
load(
"//tensorflow:tensorflow.bzl",
"VERSION",
"tf_binary_additional_srcs",
"tf_cc_binary",
"tf_cc_test",
"tf_copts",
"tf_custom_op_library",
"tf_java_test",
)
load(":build_defs.bzl", "JAVACOPTS")
load(":src/gen/gen_ops.bzl", "tf_java_op_gen_srcjar")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//visibility:private"],
licenses = ["notice"],
)
java_library(
name = "tensorflow",
srcs = [
":java_op_sources",
":java_sources",
],
data = tf_binary_additional_srcs() + [":libtensorflow_jni"],
javacopts = JAVACOPTS,
plugins = [":processor"],
resources = [":java_resources"],
visibility = ["//visibility:public"],
)
genrule(
name = "version-info",
outs = ["src/main/resources/tensorflow-version-info"],
cmd = "echo version=%s > $@" % VERSION,
output_to_bindir = 1,
)
filegroup(
name = "java_resources",
srcs = [":version-info"],
visibility = ["//visibility:private"],
)
# NOTE(ashankar): Rule to include the Java API in the Android Inference Library
# .aar. At some point, might make sense for a .aar rule here instead.
filegroup(
name = "java_sources",
srcs = glob([
"src/main/java/org/tensorflow/*.java",
"src/main/java/org/tensorflow/types/*.java",
]),
visibility = ["//tensorflow/tools/android/inference_interface:__pkg__"],
)
java_plugin(
name = "processor",
generates_api = True,
processor_class = "org.tensorflow.processor.OperatorProcessor",
visibility = ["//visibility:public"],
deps = [":processor_library"],
)
java_library(
name = "processor_library",
srcs = glob(["src/gen/java/org/tensorflow/processor/**/*.java"]),
javacopts = JAVACOPTS,
resources = glob(["src/gen/resources/META-INF/services/javax.annotation.processing.Processor"]),
deps = [
"@com_google_guava",
"@com_squareup_javapoet",
],
)
filegroup(
name = "java_op_sources",
srcs = glob(["src/main/java/org/tensorflow/op/**/*.java"]) + [":java_op_gen_sources"],
visibility = ["//visibility:private"],
)
tf_java_op_gen_srcjar(
name = "java_op_gen_sources",
api_def_srcs = [
"//tensorflow/core/api_def:base_api_def",
"//tensorflow/core/api_def:java_api_def",
],
base_package = "org.tensorflow.op",
gen_tool = ":java_op_gen_tool",
)
tf_cc_binary(
name = "java_op_gen_tool",
srcs = [
"src/gen/cc/op_gen_main.cc",
],
copts = tf_copts(),
linkopts = select({
"//tensorflow:windows": [],
"//conditions:default": ["-lm"],
}),
linkstatic = 1,
deps = [
":java_op_gen_lib",
"//tensorflow/core:framework",
"//tensorflow/core:framework_internal",
"//tensorflow/core:lib",
"//tensorflow/core:ops",
"//tensorflow/core:protos_all_cc",
"@com_google_absl//absl/log:check",
"@xla//xla/tsl/platform:status",
],
)
cc_library(
name = "java_op_gen_lib",
srcs = [
"src/gen/cc/op_generator.cc",
"src/gen/cc/op_specs.cc",
"src/gen/cc/source_writer.cc",
],
hdrs = [
"src/gen/cc/java_defs.h",
"src/gen/cc/op_generator.h",
"src/gen/cc/op_specs.h",
"src/gen/cc/source_writer.h",
],
copts = tf_copts(),
deps = [
"//tensorflow/core:framework",
"//tensorflow/core:framework_internal",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:op_gen_lib",
"//tensorflow/core:protos_all_cc",
"@com_google_absl//absl/log",
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@com_googlesource_code_re2//:re2",
],
)
java_library(
name = "testutil",
testonly = 1,
srcs = ["src/test/java/org/tensorflow/TestUtil.java"],
javacopts = JAVACOPTS,
deps = [":tensorflow"],
)
tf_java_test(
name = "EagerSessionTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/EagerSessionTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.EagerSessionTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_java_test(
name = "EagerOperationBuilderTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/EagerOperationBuilderTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.EagerOperationBuilderTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_java_test(
name = "EagerOperationTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/EagerOperationTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.EagerOperationTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_java_test(
name = "GraphTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/GraphTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.GraphTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_java_test(
name = "GraphOperationBuilderTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/GraphOperationBuilderTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.GraphOperationBuilderTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_java_test(
name = "GraphOperationTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/GraphOperationTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.GraphOperationTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_java_test(
name = "SavedModelBundleTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/SavedModelBundleTest.java"],
data = ["//tensorflow/cc/saved_model:saved_model_half_plus_two"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.SavedModelBundleTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_java_test(
name = "SessionTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/SessionTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.SessionTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_java_test(
name = "ShapeTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/ShapeTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.ShapeTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_custom_op_library(
name = "my_test_op.so",
srcs = ["src/test/native/my_test_op.cc"],
)
tf_java_test(
name = "TensorFlowTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/TensorFlowTest.java"],
data = [":my_test_op.so"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.TensorFlowTest",
deps = [
":tensorflow",
"@junit",
],
)
tf_java_test(
name = "TensorTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/TensorTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.TensorTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_java_test(
name = "ScopeTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/op/ScopeTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.op.ScopeTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_java_test(
name = "PrimitiveOpTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/op/PrimitiveOpTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.op.PrimitiveOpTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_java_test(
name = "OperandsTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/op/OperandsTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.op.OperandsTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_java_test(
name = "ConstantTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/op/core/ConstantTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.op.core.ConstantTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_java_test(
name = "GeneratedOperationsTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.op.core.GeneratedOperationsTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_java_test(
name = "GradientsTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/op/core/GradientsTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.op.core.GradientsTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
tf_java_test(
name = "ZerosTest",
size = "small",
srcs = ["src/test/java/org/tensorflow/op/core/ZerosTest.java"],
javacopts = JAVACOPTS,
test_class = "org.tensorflow.op.core.ZerosTest",
deps = [
":tensorflow",
":testutil",
"@junit",
],
)
filegroup(
name = "processor_test_resources",
srcs = glob([
"src/test/resources/org/tensorflow/**/*.java",
"src/main/java/org/tensorflow/op/annotation/Operator.java",
]),
)
tf_cc_test(
name = "source_writer_test",
size = "small",
srcs = [
"src/gen/cc/source_writer_test.cc",
],
data = [
"src/gen/resources/test.java.snippet",
],
deps = [
":java_op_gen_lib",
"//tensorflow/core:lib",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
],
)
filegroup(
name = "libtensorflow_jni",
srcs = select({
"//tensorflow:windows": [":tensorflow_jni.dll"],
"//tensorflow:macos": [":libtensorflow_jni.dylib"],
"//conditions:default": [":libtensorflow_jni.so"],
}),
visibility = ["//visibility:public"],
)
LINKER_VERSION_SCRIPT = ":config/version_script.lds"
LINKER_EXPORTED_SYMBOLS = ":config/exported_symbols.lds"
tf_cc_binary(
name = "tensorflow_jni",
# Set linker options to strip out anything except the JNI
# symbols from the library. This reduces the size of the library
# considerably (~50% as of January 2017).
linkopts = select({
"//tensorflow:debug": [], # Disable all custom linker options in debug mode
"//tensorflow:macos": [
"-Wl,-exported_symbols_list,$(location {})".format(LINKER_EXPORTED_SYMBOLS),
],
"//tensorflow:windows": [],
"//conditions:default": [
"-z defs",
"-s",
"-Wl,--version-script,$(location {})".format(LINKER_VERSION_SCRIPT),
# copybara:uncomment_begin(google-only)
# "-Wl,--undefined-version",
# copybara:uncomment_end
],
}),
linkshared = 1,
linkstatic = 1,
per_os_targets = True,
deps = [
"//tensorflow/core/distributed_runtime/rpc:grpc_server_lib",
"//tensorflow/java/src/main/native",
LINKER_VERSION_SCRIPT,
LINKER_EXPORTED_SYMBOLS,
],
)
genrule(
name = "pom",
outs = ["pom.xml"],
cmd = "$(location generate_pom) >$@",
output_to_bindir = 1,
tools = [":generate_pom"] + tf_binary_additional_srcs(),
)
tf_cc_binary(
name = "generate_pom",
srcs = ["generate_pom.cc"],
deps = ["//tensorflow/c:c_api"],
)
tensorflow/java/LEGACY.md
+85
View File
@@ -0,0 +1,85 @@
## Quickstart
- Refer to
[Installing TensorFlow for Java](https://www.tensorflow.org/install/lang_java)
- [Javadoc](https://www.tensorflow.org/api_docs/java/reference/org/tensorflow/package-summary)
- [![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.tensorflow/tensorflow/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.tensorflow/tensorflow)
## Nightly builds
Releases built from release branches are available on Maven Central.
Additionally, every day binaries are built from the `master` branch on GitHub:
- [JAR](https://storage.googleapis.com/tensorflow-nightly/github/tensorflow/lib_package/libtensorflow.jar)
- [Source JAR](https://storage.googleapis.com/tensorflow-nightly/github/tensorflow/lib_package/libtensorflow-src.jar)
- JNI:
- [Linux CPU-only](https://storage.googleapis.com/tensorflow-nightly/github/tensorflow/lib_package/libtensorflow_jni-cpu-linux-x86_64.tar.gz)
- [Linux GPU](https://storage.googleapis.com/tensorflow-nightly/github/tensorflow/lib_package/libtensorflow_jni-gpu-linux-x86_64.tar.gz)
- [MacOS](https://storage.googleapis.com/tensorflow-nightly/github/tensorflow/lib_package/libtensorflow_jni-cpu-darwin-x86_64.tar.gz)
- Windows: (No nightly builds available yet)
## Building from source
If the quickstart instructions above do not work out, the TensorFlow Java and
native libraries will need to be built from source.
1. Install [bazel](https://www.bazel.build/versions/master/docs/install.html)
2. Setup the environment to build TensorFlow from source code
([Linux or macOS](https://www.tensorflow.org/install/source)). If you'd like
to skip reading those details and do not care about GPU support, try the
following:
```sh
# On Linux
sudo apt-get install python swig python-numpy
# On Mac OS X with homebrew
brew install swig
```
3. [Configure](https://www.tensorflow.org/install/source) (e.g., enable GPU
support) and build:
```sh
./configure
bazel build --config opt \
//tensorflow/java:tensorflow \
//tensorflow/java:libtensorflow_jni
```
The command above will produce two files in the `bazel-bin/tensorflow/java`
directory:
* An archive of Java classes: `libtensorflow.jar`
* A native library: `libtensorflow_jni.so` on Linux, `libtensorflow_jni.dylib`
on OS X, or `tensorflow_jni.dll` on Windows.
To compile Java code that uses the TensorFlow Java API, include
`libtensorflow.jar` in the classpath. For example:
```sh
javac -cp bazel-bin/tensorflow/java/libtensorflow.jar ...
```
To execute the compiled program, include `libtensorflow.jar` in the classpath
and the native library in the library path. For example:
```sh
java -cp bazel-bin/tensorflow/java/libtensorflow.jar \
-Djava.library.path=bazel-bin/tensorflow/java \
...
```
Installation on Windows requires the more experimental
[bazel on Windows](https://bazel.build/versions/master/docs/windows.html).
### Bazel
If your project uses bazel for builds, add a dependency on
`//tensorflow/java:tensorflow` to the `java_binary` or `java_library` rule. For
example:
```sh
bazel run -c opt //tensorflow/java/src/main/java/org/tensorflow/examples:label_image
```
tensorflow/java/README.md
+9
View File
@@ -0,0 +1,9 @@
# TensorFlow for Java
> *WARNING*: This version of the Java client is deprecated and has been replaced by a new version that is maintained
> in its own [repository](https://github.com/tensorflow/java) under the TensorFlow organization.
>
> For using TensorFlow on a JVM, please refer to new [TensorFlow Java](https://www.tensorflow.org/jvm/install).
> For using TensorFlow on Android, refer instead to [TensorFlow Lite](https://www.tensorflow.org/code/tensorflow/lite/).
Follow this [link](LEGACY.md) for legacy instructions.
tensorflow/java/build_defs.bzl
+146
View File
@@ -0,0 +1,146 @@
JAVA_VERSION_OPTS = []
# A more robust set of lint and errorprone checks when building
# Java source to improve code consistency.
# The bazel errorprone plugin currently only enables default errorChecks
# https://github.com/bazelbuild/bazel/blob/97975603e5ff2247e6bb352e3afd27fea38f108d/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/errorprone/ErrorPronePlugin.java#L52
#
# Default errorChecks are errorprone checkers listed under ENABLED_ERRORS at
# https://github.com/google/error-prone/blob/c6f24bc387989158d99af28e7ae86755e56c5f38/core/src/main/java/com/google/errorprone/scanner/BuiltInCheckerSuppliers.java#L273
#
# Here we enable all available errorprone checks to converge on a consistent
# code style.
# https://github.com/google/error-prone/blob/c6f24bc387989158d99af28e7ae86755e56c5f38/core/src/main/java/com/google/errorprone/scanner/BuiltInCheckerSuppliers.java#L260
# This list is from ENABLED_WARNINGS in
# com/google/errorprone/scanner/BuiltInCheckerSuppliers.java
EP_ENABLED_WARNINGS = [
"-Xep:AmbiguousMethodReference:WARN",
"-Xep:ArgumentSelectionDefectChecker:WARN",
"-Xep:AssertEqualsArgumentOrderChecker:WARN",
"-Xep:BadAnnotationImplementation:WARN",
"-Xep:BadComparable:WARN",
"-Xep:BoxedPrimitiveConstructor:WARN",
"-Xep:CannotMockFinalClass:WARN",
"-Xep:ClassCanBeStatic:WARN",
"-Xep:ClassNewInstance:WARN",
"-Xep:DefaultCharset:WARN",
"-Xep:DoubleCheckedLocking:WARN",
"-Xep:ElementsCountedInLoop:WARN",
"-Xep:EqualsHashCode:WARN",
"-Xep:EqualsIncompatibleType:WARN",
"-Xep:Finally:WARN",
"-Xep:FloatingPointLiteralPrecision:WARN",
"-Xep:FragmentInjection:WARN",
"-Xep:FragmentNotInstantiable:WARN",
"-Xep:FunctionalInterfaceClash:WARN",
"-Xep:FutureReturnValueIgnored:WARN",
"-Xep:GetClassOnEnum:WARN",
"-Xep:ImmutableAnnotationChecker:WARN",
"-Xep:ImmutableEnumChecker:WARN",
"-Xep:IncompatibleModifiers:WARN",
"-Xep:InjectOnConstructorOfAbstractClass:WARN",
"-Xep:InputStreamSlowMultibyteRead:WARN",
"-Xep:IterableAndIterator:WARN",
"-Xep:JavaLangClash:WARN",
"-Xep:JUnit3FloatingPointComparisonWithoutDelta:WARN",
"-Xep:JUnitAmbiguousTestClass:WARN",
"-Xep:LiteralClassName:WARN",
"-Xep:LogicalAssignment:WARN",
"-Xep:MissingFail:WARN",
"-Xep:MissingOverride:WARN",
"-Xep:MutableConstantField:WARN",
"-Xep:NamedParameters:WARN",
"-Xep:NarrowingCompoundAssignment:WARN",
"-Xep:NonAtomicVolatileUpdate:WARN",
"-Xep:NonOverridingEquals:WARN",
"-Xep:NullableConstructor:WARN",
"-Xep:NullablePrimitive:WARN",
"-Xep:NullableVoid:WARN",
"-Xep:OperatorPrecedence:WARN",
"-Xep:OverridesGuiceInjectableMethod:WARN",
"-Xep:PreconditionsInvalidPlaceholder:WARN",
"-Xep:ProtoFieldPreconditionsCheckNotNull:WARN",
"-Xep:ReferenceEquality:WARN",
"-Xep:RequiredModifiers:WARN",
"-Xep:ShortCircuitBoolean:WARN",
"-Xep:SimpleDateFormatConstant:WARN",
"-Xep:StaticGuardedByInstance:WARN",
"-Xep:SynchronizeOnNonFinalField:WARN",
"-Xep:TruthConstantAsserts:WARN",
"-Xep:TypeParameterShadowing:WARN",
"-Xep:TypeParameterUnusedInFormals:WARN",
"-Xep:UnsynchronizedOverridesSynchronized:WARN",
"-Xep:URLEqualsHashCode:WARN",
"-Xep:WaitNotInLoop:WARN",
]
# This list is from DISABLED_CHECKS in
# com/google/errorprone/scanner/BuiltInCheckerSuppliers.java
EP_DISABLED_CHECKS = [
"-Xep:AutoFactoryAtInject:WARN",
"-Xep:AssertFalse:WARN",
"-Xep:AssistedInjectAndInjectOnConstructors:WARN",
"-Xep:AssistedInjectAndInjectOnSameConstructor:WARN",
"-Xep:BigDecimalLiteralDouble:WARN",
"-Xep:BindingToUnqualifiedCommonType:WARN",
"-Xep:ClassName:WARN",
"-Xep:ComparisonContractViolated:WARN",
"-Xep:ConstantField:WARN",
"-Xep:ConstructorInvokesOverridable:WARN",
# False positives, disabled
# "-Xep:ConstructorLeaksThis:WARN",
"-Xep:DepAnn:WARN",
"-Xep:DivZero:WARN",
"-Xep:EmptyIfStatement:WARN",
"-Xep:EmptySetMultibindingContributions:WARN",
"-Xep:EmptyTopLevelDeclaration:WARN",
"-Xep:ExpectedExceptionChecker:WARN",
"-Xep:HardCodedSdCardPath:WARN",
"-Xep:InjectedConstructorAnnotations:WARN",
"-Xep:InvalidTargetingOnScopingAnnotation:WARN",
"-Xep:IterablePathParameter:WARN",
"-Xep:JMockTestWithoutRunWithOrRuleAnnotation:WARN",
"-Xep:JavaxInjectOnFinalField:WARN",
"-Xep:LockMethodChecker:WARN",
"-Xep:LongLiteralLowerCaseSuffix:WARN",
"-Xep:MethodCanBeStatic:WARN",
"-Xep:MissingDefault:WARN",
"-Xep:MixedArrayDimensions:WARN",
"-Xep:MoreThanOneQualifier:WARN",
"-Xep:MultiVariableDeclaration:WARN",
"-Xep:MultipleTopLevelClasses:WARN",
"-Xep:NoAllocationChecker:WARN",
"-Xep:NonCanonicalStaticMemberImport:WARN",
"-Xep:NumericEquality:WARN",
"-Xep:PackageLocation:WARN",
"-Xep:PrimitiveArrayPassedToVarargsMethod:WARN",
"-Xep:PrivateConstructorForUtilityClass:WARN",
"-Xep:PrivateConstructorForNoninstantiableModule:WARN",
"-Xep:ProtoStringFieldReferenceEquality:WARN",
"-Xep:QualifierOrScopeOnInjectMethod:WARN",
"-Xep:QualifierWithTypeUse:WARN",
"-Xep:RedundantThrows:WARN",
"-Xep:RemoveUnusedImports:WARN",
"-Xep:ScopeAnnotationOnInterfaceOrAbstractClass:WARN",
"-Xep:ScopeOrQualifierAnnotationRetention:WARN",
"-Xep:StaticQualifiedUsingExpression:WARN",
"-Xep:StaticOrDefaultInterfaceMethod:WARN",
"-Xep:StringEquality:WARN",
"-Xep:TestExceptionChecker:WARN",
# TODO: stylistic changes in code
# "-Xep:ThrowsUncheckedException:WARN",
# "-Xep:UngroupedOverloads:WARN",
"-Xep:UnlockMethodChecker:WARN",
"-Xep:UnnecessaryDefaultInEnumSwitch:WARN",
"-Xep:UnnecessaryStaticImport:WARN",
"-Xep:UseBinds:WARN",
"-Xep:VarChecker:WARN",
"-Xep:WildcardImport:WARN",
"-Xep:WrongParameterPackage:WARN",
]
EP_OPTS = EP_ENABLED_WARNINGS + EP_DISABLED_CHECKS
JAVACOPTS = JAVA_VERSION_OPTS + EP_OPTS
tensorflow/java/config/exported_symbols.lds
+3
View File
@@ -0,0 +1,3 @@
*Java_org_tensorflow_*
*JNI_OnLoad
*JNI_OnUnload
tensorflow/java/config/version_script.lds
+11
View File
@@ -0,0 +1,11 @@
VERS_1.0 {
# Export JNI symbols.
global:
Java_*;
JNI_OnLoad;
JNI_OnUnload;
# Hide everything else.
local:
*;
};
tensorflow/java/generate_pom.cc
+61
View File
@@ -0,0 +1,61 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <cstddef>
#include <iostream>
#include <string>
#include "tensorflow/c/c_api.h"
int main(int argc, char** argv) {
std::string tmpl(R"EOF(
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<groupId>org.tensorflow</groupId>
<artifactId>libtensorflow</artifactId>
<version>{{TENSORFLOW_VERSION}}</version>
<packaging>jar</packaging>
<name>tensorflow</name>
<url>https://www.tensorflow.org</url>
<inceptionYear>2015</inceptionYear>
<licenses>
<license>
<name>The Apache Software License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
<distribution>repo</distribution>
</license>
</licenses>
<scm>
<url>https://github.com/tensorflow/tensorflow.git</url>
<connection>git@github.com:tensorflow/tensorflow.git</connection>
<developerConnection>scm:git:https://github.com/tensorflow/tensorflow.git</developerConnection>
</scm>
</project>
)EOF");
const std::string var("{{TENSORFLOW_VERSION}}");
const std::string val(TF_Version());
for (size_t pos = tmpl.find(var); pos != std::string::npos;
pos = tmpl.find(var)) {
tmpl.replace(pos, var.size(), val);
}
std::cout << tmpl;
return 0;
}
tensorflow/java/src/gen/cc/java_defs.h
+290
View File
@@ -0,0 +1,290 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_GEN_CC_JAVA_DEFS_H_
#define TENSORFLOW_JAVA_SRC_GEN_CC_JAVA_DEFS_H_
#include <list>
#include <map>
#include <string>
#include <utility>
#include "tensorflow/core/framework/types.h"
namespace tensorflow {
namespace java {
// An enumeration of different modifiers commonly used in Java
enum Modifier {
PACKAGE = 0,
PUBLIC = (1 << 0),
PROTECTED = (1 << 1),
PRIVATE = (1 << 2),
STATIC = (1 << 3),
FINAL = (1 << 4),
};
class Annotation;
// A definition of any kind of Java type (classes, interfaces...)
//
// Note that most of the data fields of this class are only useful in specific
// contexts and are not required in many cases. For example, annotations and
// supertypes are only useful when declaring a type.
class Type {
public:
enum Kind {
PRIMITIVE, CLASS, INTERFACE, ENUM, GENERIC, ANNOTATION
};
static const Type Byte() {
return Type(Type::PRIMITIVE, "byte");
}
static const Type Char() {
return Type(Type::PRIMITIVE, "char");
}
static const Type Short() {
return Type(Type::PRIMITIVE, "short");
}
static const Type Int() {
return Type(Type::PRIMITIVE, "int");
}
static const Type Long() {
return Type(Type::PRIMITIVE, "long");
}
static const Type Float() {
return Type(Type::PRIMITIVE, "float");
}
static const Type Double() {
return Type(Type::PRIMITIVE, "double");
}
static const Type Boolean() {
return Type(Type::PRIMITIVE, "boolean");
}
static const Type Void() {
// For simplicity, we consider 'void' as a primitive type, like the Java
// Reflection API does
return Type(Type::PRIMITIVE, "void");
}
static Type Generic(const std::string& name) {
return Type(Type::GENERIC, name);
}
static Type Wildcard() { return Type(Type::GENERIC, ""); }
static Type Class(const std::string& name, const std::string& package = "") {
return Type(Type::CLASS, name, package);
}
static Type Interface(const std::string& name,
const std::string& package = "") {
return Type(Type::INTERFACE, name, package);
}
static Type Enum(const std::string& name, const std::string& package = "") {
return Type(Type::ENUM, name, package);
}
static Type ClassOf(const Type& type) {
return Class("Class").add_parameter(type);
}
static Type ListOf(const Type& type) {
return Interface("List", "java.util").add_parameter(type);
}
static Type IterableOf(const Type& type) {
return Interface("Iterable").add_parameter(type);
}
static Type ForDataType(DataType data_type) {
switch (data_type) {
case DataType::DT_BOOL:
return Class("Boolean");
case DataType::DT_STRING:
return Class("String");
case DataType::DT_FLOAT:
return Class("Float");
case DataType::DT_DOUBLE:
return Class("Double");
case DataType::DT_UINT8:
return Class("UInt8", "org.tensorflow.types");
case DataType::DT_INT32:
return Class("Integer");
case DataType::DT_INT64:
return Class("Long");
case DataType::DT_RESOURCE:
// TODO(karllessard) create a Resource utility class that could be
// used to store a resource and its type (passed in a second argument).
// For now, we need to force a wildcard and we will unfortunately lose
// track of the resource type.
// Falling through...
default:
// Any other datatypes does not have a equivalent in Java and must
// remain a wildcard (e.g. DT_COMPLEX64, DT_QINT8, ...)
return Wildcard();
}
}
const Kind& kind() const { return kind_; }
const std::string& name() const { return name_; }
const std::string& package() const { return package_; }
const std::string canonical_name() const {
return package_.empty() ? name_ : package_ + "." + name_;
}
bool wildcard() const { return name_.empty(); } // only wildcards has no name
const std::list<Type>& parameters() const { return parameters_; }
Type& add_parameter(const Type& parameter) {
parameters_.push_back(parameter);
return *this;
}
const std::list<Annotation>& annotations() const { return annotations_; }
Type& add_annotation(const Annotation& annotation) {
annotations_.push_back(annotation);
return *this;
}
const std::list<Type>& supertypes() const { return supertypes_; }
Type& add_supertype(const Type& type) {
if (type.kind_ == CLASS) {
supertypes_.push_front(type); // keep superclass at the front of the list
} else if (type.kind_ == INTERFACE) {
supertypes_.push_back(type);
}
return *this;
}
protected:
Type(Kind kind, const std::string& name, const std::string& package = "")
: kind_(kind), name_(name), package_(package) {}
private:
Kind kind_;
std::string name_;
std::string package_;
std::list<Type> parameters_;
std::list<Annotation> annotations_;
std::list<Type> supertypes_;
};
// Definition of a Java annotation
//
// This class only defines the usage of an annotation in a specific context,
// giving optionally a set of attributes to initialize.
class Annotation : public Type {
public:
static Annotation Create(const std::string& type_name,
const std::string& pkg = "") {
return Annotation(type_name, pkg);
}
const std::string& attributes() const { return attributes_; }
Annotation& attributes(const std::string& attributes) {
attributes_ = attributes;
return *this;
}
private:
std::string attributes_;
Annotation(const std::string& name, const std::string& package)
: Type(Kind::ANNOTATION, name, package) {}
};
// A definition of a Java variable
//
// This class declares an instance of a type, such as a class field or a
// method argument, which can be documented.
class Variable {
public:
static Variable Create(const std::string& name, const Type& type) {
return Variable(name, type, false);
}
static Variable Varargs(const std::string& name, const Type& type) {
return Variable(name, type, true);
}
const std::string& name() const { return name_; }
const Type& type() const { return type_; }
bool variadic() const { return variadic_; }
private:
std::string name_;
Type type_;
bool variadic_;
Variable(const std::string& name, const Type& type, bool variadic)
: name_(name), type_(type), variadic_(variadic) {}
};
// A definition of a Java class method
//
// This class defines the signature of a method, including its name, return
// type and arguments.
class Method {
public:
static Method Create(const std::string& name, const Type& return_type) {
return Method(name, return_type, false);
}
static Method ConstructorFor(const Type& clazz) {
return Method(clazz.name(), clazz, true);
}
bool constructor() const { return constructor_; }
const std::string& name() const { return name_; }
const Type& return_type() const { return return_type_; }
const std::list<Variable>& arguments() const { return arguments_; }
Method& add_argument(const Variable& var) {
arguments_.push_back(var);
return *this;
}
const std::list<Annotation>& annotations() const { return annotations_; }
Method& add_annotation(const Annotation& annotation) {
annotations_.push_back(annotation);
return *this;
}
private:
std::string name_;
Type return_type_;
bool constructor_;
std::list<Variable> arguments_;
std::list<Annotation> annotations_;
Method(const std::string& name, const Type& return_type, bool constructor)
: name_(name), return_type_(return_type), constructor_(constructor) {}
};
// A definition of a documentation bloc for a Java element (JavaDoc)
class Javadoc {
public:
static Javadoc Create(const std::string& brief = "") {
return Javadoc(brief);
}
const std::string& brief() const { return brief_; }
const std::string& details() const { return details_; }
Javadoc& details(const std::string& details) {
details_ = details;
return *this;
}
const std::list<std::pair<std::string, std::string>>& tags() const {
return tags_;
}
Javadoc& add_tag(const std::string& tag, const std::string& text) {
tags_.push_back(std::make_pair(tag, text));
return *this;
}
Javadoc& add_param_tag(const std::string& name, const std::string& text) {
return add_tag("param", name + " " + text);
}
private:
std::string brief_;
std::string details_;
std::list<std::pair<std::string, std::string>> tags_;
explicit Javadoc(const std::string& brief) : brief_(brief) {}
};
} // namespace java
} // namespace tensorflow
#endif // TENSORFLOW_JAVA_SRC_GEN_CC_JAVA_DEFS_H_
tensorflow/java/src/gen/cc/op_gen_main.cc
+79
View File
@@ -0,0 +1,79 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <vector>
#include "absl/log/check.h"
#include "xla/tsl/platform/status.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/op_def.pb.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/platform/init_main.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/util/command_line_flags.h"
#include "tensorflow/java/src/gen/cc/op_generator.h"
namespace tensorflow {
namespace java {
const char kUsageHeader[] =
"\n\nGenerator of operation wrappers in Java.\n\n"
"This executable generates wrappers for all registered operations it has "
"been compiled with. A wrapper exposes an intuitive and strongly-typed\n"
"interface for building its underlying operation and linking it into a "
"graph.\n\n"
"Operation wrappers are generated under the path specified by the "
"'--output_dir' argument. This path can be absolute or relative to the\n"
"current working directory and will be created if it does not exist.\n\n"
"Note that the operations will not be available through the "
"'org.tensorflow.op.Ops' API until the generated classes are compiled\n"
"using an appropriate annotation processor.\n\n"
"The '--base_package' overrides the default parent package under which "
"the generated subpackage and classes are to be located.\n\n"
"Finally, the `--api_dirs` argument takes a list of comma-separated "
"directories of API definitions can be provided to override default\n"
"values found in the ops definitions. Directories are ordered by priority "
"(the last having precedence over the first).\n\n";
} // namespace java
} // namespace tensorflow
int main(int argc, char* argv[]) {
std::string output_dir;
std::string base_package = "org.tensorflow.op";
std::string api_dirs_str;
std::vector<tensorflow::Flag> flag_list = {
tensorflow::Flag("output_dir", &output_dir,
"Root directory into which output files are generated"),
tensorflow::Flag(
"base_package", &base_package,
"Package parent to the generated subpackage and classes"),
tensorflow::Flag(
"api_dirs", &api_dirs_str,
"List of directories that contains the ops api definitions")};
std::string usage = tensorflow::java::kUsageHeader;
usage += tensorflow::Flags::Usage(argv[0], flag_list);
bool parsed_flags_ok = tensorflow::Flags::Parse(&argc, argv, flag_list);
tensorflow::port::InitMain(usage.c_str(), &argc, &argv);
QCHECK(parsed_flags_ok && !output_dir.empty()) << usage;
std::vector<std::string> api_dirs = tensorflow::str_util::Split(
api_dirs_str, ",", tensorflow::str_util::SkipEmpty());
tensorflow::java::OpGenerator generator(api_dirs);
tensorflow::OpList ops;
tensorflow::OpRegistry::Global()->Export(false, &ops);
TF_CHECK_OK(generator.Run(ops, base_package, output_dir));
return 0;
}
tensorflow/java/src/gen/cc/op_generator.cc
+572
View File
@@ -0,0 +1,572 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/java/src/gen/cc/op_generator.h"
#include <list>
#include <map>
#include <memory>
#include <set>
#include <sstream>
#include <string>
#include <utility>
#include <vector>
#include "absl/status/status.h"
#include "absl/strings/ascii.h"
#include "xla/tsl/platform/status.h"
#include "tensorflow/core/framework/api_def.pb.h"
#include "tensorflow/core/framework/op_def.pb.h"
#include "tensorflow/core/framework/op_gen_lib.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/file_system.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/path.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/java/src/gen/cc/java_defs.h"
#include "tensorflow/java/src/gen/cc/op_specs.h"
#include "tensorflow/java/src/gen/cc/source_writer.h"
namespace tensorflow {
namespace java {
namespace {
constexpr const char kLicense[] =
"/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n"
"\n"
"Licensed under the Apache License, Version 2.0 (the \"License\");\n"
"you may not use this file except in compliance with the License.\n"
"You may obtain a copy of the License at\n"
"\n"
" http://www.apache.org/licenses/LICENSE-2.0\n"
"\n"
"Unless required by applicable law or agreed to in writing, software\n"
"distributed under the License is distributed on an \"AS IS\" BASIS,\n"
"WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n"
"See the License for the specific language governing permissions and\n"
"limitations under the License.\n"
"=======================================================================*/"
"\n";
// There is three different modes to render an op class, depending on the
// number and type of outputs it has:
//
// DEFAULT: This mode does not provide any specialization for the op class, it
// is applied when the operation does not comply with any other mode
//
// OPERAND: The op class implements the Operand<T> interface, allowing an
// instance to be passed directly in input to another operation
//
// LIST_OPERAND: The op class implements the Iterable<Operand<T>> interface,
// allowing an instance to be passed directly as a list input to
// another operation
//
enum RenderMode { DEFAULT, OPERAND, LIST_OPERAND };
void AddArgument(const Variable& var, const std::string& description,
Method* method_out, Javadoc* javadoc_out) {
method_out->add_argument(var);
javadoc_out->add_param_tag(var.name(), description);
}
void CollectOpDependencies(const OpSpec& op, RenderMode mode,
std::list<Type>* out) {
out->push_back(Type::Class("Operation", "org.tensorflow"));
out->push_back(Type::Class("OperationBuilder", "org.tensorflow"));
out->push_back(Type::Class("Scope", "org.tensorflow.op"));
if (mode == OPERAND) {
out->push_back(Type::Class("Output", "org.tensorflow"));
} else if (mode == LIST_OPERAND) {
out->push_back(Type::Interface("Iterator", "java.util"));
}
// Don't pay attention to duplicate types in the dependency list, they will
// be filtered out by the SourceWriter.
for (const ArgumentSpec& input : op.inputs()) {
out->push_back(input.var().type());
if (input.iterable()) {
out->push_back(Type::Class("Operands", "org.tensorflow.op"));
}
}
for (const ArgumentSpec& output : op.outputs()) {
out->push_back(output.var().type());
if (output.iterable()) {
out->push_back(Type::Class("Arrays", "java.util"));
}
}
for (const AttributeSpec& attribute : op.attributes()) {
out->push_back(attribute.var().type());
out->push_back(attribute.jni_type());
if (attribute.has_default_value() &&
attribute.type().kind() == Type::GENERIC) {
out->push_back(Type::ForDataType(attribute.default_value()->type()));
}
}
for (const AttributeSpec& optional_attribute : op.optional_attributes()) {
out->push_back(optional_attribute.var().type());
}
}
void WriteSetAttrDirective(const AttributeSpec& attr, bool optional,
SourceWriter* writer) {
std::string var_name =
optional ? "opts." + attr.var().name() : attr.var().name();
if (attr.iterable()) {
std::string array_name = attr.var().name() + "Array";
writer->AppendType(attr.jni_type())
.Append("[] " + array_name + " = new ")
.AppendType(attr.jni_type())
.Append("[" + var_name + ".size()];")
.EndLine()
.BeginBlock("for (int i = 0; i < " + array_name + ".length; ++i)")
.Append(array_name + "[i] = ");
if (attr.type().kind() == Type::GENERIC) {
writer->Append("DataType.fromClass(" + var_name + ".get(i));");
} else {
writer->Append(var_name + ".get(i);");
}
writer->EndLine()
.EndBlock()
.Append("opBuilder.setAttr(\"" + attr.op_def_name() + "\", ")
.Append(array_name + ");")
.EndLine();
} else {
writer->Append("opBuilder.setAttr(\"" + attr.op_def_name() + "\", ");
if (attr.var().type().name() == "Class") {
writer->Append("DataType.fromClass(" + var_name + "));");
} else {
writer->Append(var_name + ");");
}
writer->EndLine();
}
}
void RenderSecondaryFactoryMethod(const OpSpec& op, const Type& op_class,
std::map<std::string, Type> default_types,
SourceWriter* writer) {
// Build the return type for the secondary factory, replacing generic
// parameters with their default value if any
Type return_type = Type::Class(op_class.name(), op_class.package());
for (const Type& parameter : op_class.parameters()) {
if (parameter.kind() == Type::GENERIC &&
default_types.find(parameter.name()) != default_types.end()) {
return_type.add_parameter(default_types.at(parameter.name()));
} else {
return_type.add_parameter(parameter);
}
}
Method factory = Method::Create("create", return_type);
Javadoc factory_doc = Javadoc::Create(
"Factory method to create a class wrapping a new " + op_class.name() +
" operation using default output types.");
Variable scope =
Variable::Create("scope", Type::Class("Scope", "org.tensorflow.op"));
AddArgument(scope, "current scope", &factory, &factory_doc);
std::stringstream factory_statement;
factory_statement << "return create(scope";
for (const ArgumentSpec& input : op.inputs()) {
AddArgument(input.var(), input.description(), &factory, &factory_doc);
factory_statement << ", " << input.var().name();
}
for (const AttributeSpec& attr : op.attributes()) {
// Only add attributes that are not types or have no default value to the
// signature of the secondary factory
factory_statement << ", ";
if (attr.type().kind() == Type::GENERIC &&
default_types.find(attr.type().name()) != default_types.end()) {
factory_statement << default_types.at(attr.type().name()).name()
<< ".class";
} else {
AddArgument(attr.var(), attr.description(), &factory, &factory_doc);
factory_statement << attr.var().name();
}
}
if (!op.optional_attributes().empty()) {
Variable options_var = Variable::Varargs("options", Type::Class("Options"));
AddArgument(options_var, "carries optional attributes values", &factory,
&factory_doc);
factory_statement << ", " << options_var.name();
}
factory_doc.add_tag("return", "a new instance of " + op_class.name());
writer->BeginMethod(factory, PUBLIC | STATIC, &factory_doc);
writer->Append(factory_statement.str()).Append(");").EndLine();
writer->EndMethod();
}
void RenderFactoryMethods(const OpSpec& op, const Type& op_class,
SourceWriter* writer) {
Method factory = Method::Create("create", op_class);
Javadoc factory_doc =
Javadoc::Create("Factory method to create a class wrapping a new " +
op_class.name() + " operation.");
Variable scope =
Variable::Create("scope", Type::Class("Scope", "org.tensorflow.op"));
AddArgument(scope, "current scope", &factory, &factory_doc);
for (const ArgumentSpec& input : op.inputs()) {
AddArgument(input.var(), input.description(), &factory, &factory_doc);
}
std::map<std::string, Type> default_types;
for (const AttributeSpec& attr : op.attributes()) {
AddArgument(attr.var(), attr.description(), &factory, &factory_doc);
// If this attribute is a type with a default value, save its value
// for passing it implicitly in a secondary factory method
if (attr.has_default_value() && attr.type().kind() == Type::GENERIC) {
Type default_type = Type::ForDataType(attr.default_value()->type());
if (!default_type.wildcard()) {
default_types.insert(std::make_pair(attr.type().name(), default_type));
}
}
}
if (!op.optional_attributes().empty()) {
AddArgument(Variable::Varargs("options", Type::Class("Options")),
"carries optional attributes values", &factory, &factory_doc);
}
factory_doc.add_tag("return", "a new instance of " + op_class.name());
writer->BeginMethod(factory, PUBLIC | STATIC, &factory_doc);
writer->Append("OperationBuilder opBuilder = scope.env().opBuilder(\"" +
op.graph_op_name() + "\", scope.makeOpName(\"" +
op_class.name() + "\"));");
writer->EndLine();
for (const ArgumentSpec& input : op.inputs()) {
if (input.iterable()) {
writer->Append("opBuilder.addInputList(Operands.asOutputs(" +
input.var().name() + "));");
writer->EndLine();
} else {
writer->Append("opBuilder.addInput(" + input.var().name() +
".asOutput());");
writer->EndLine();
}
}
// Add control dependencies, if any.
writer->Append("opBuilder = scope.applyControlDependencies(opBuilder);");
writer->EndLine();
for (const AttributeSpec& attribute : op.attributes()) {
WriteSetAttrDirective(attribute, false, writer);
}
if (!op.optional_attributes().empty()) {
writer->BeginBlock("if (options != null)")
.BeginBlock("for (Options opts : options)");
for (const AttributeSpec& attribute : op.optional_attributes()) {
writer->BeginBlock("if (opts." + attribute.var().name() + " != null)");
WriteSetAttrDirective(attribute, true, writer);
writer->EndBlock();
}
writer->EndBlock().EndBlock();
}
writer->Append("return new ")
.AppendType(op_class)
.Append("(opBuilder.build());")
.EndLine();
writer->EndMethod();
// If this operation has type attributes with a default value, create a
// second factory method that infers those values implicitly
if (!default_types.empty()) {
RenderSecondaryFactoryMethod(op, op_class, default_types, writer);
}
}
void RenderConstructor(const OpSpec& op, const Type& op_class,
SourceWriter* writer) {
Variable operation =
Variable::Create("operation", Type::Class("Operation", "org.tensorflow"));
Method constructor = Method::ConstructorFor(op_class).add_argument(operation);
for (const ArgumentSpec& output : op.outputs()) {
if (output.iterable() && !output.type().wildcard()) {
constructor.add_annotation(
Annotation::Create("SuppressWarnings").attributes("\"unchecked\""));
break;
}
}
writer->BeginMethod(constructor, PRIVATE)
.Append("super(operation);")
.EndLine();
if (!op.outputs().empty()) {
writer->Append("int outputIdx = 0;").EndLine();
for (const ArgumentSpec& output : op.outputs()) {
if (output.iterable()) {
std::string var_length = output.var().name() + "Length";
writer->Append("int " + var_length)
.Append(" = operation.outputListLength(\"" + output.op_def_name() +
"\");")
.EndLine()
.Append(output.var().name() + " = Arrays.asList(");
if (!output.type().wildcard()) {
writer->Append("(")
.AppendType(output.var().type().parameters().front())
.Append("[])");
}
writer->Append("operation.outputList(outputIdx, " + var_length + "));")
.EndLine()
.Append("outputIdx += " + var_length + ";")
.EndLine();
} else {
writer
->Append(output.var().name() + " = operation.output(outputIdx++);")
.EndLine();
}
}
}
writer->EndMethod();
}
void RenderGettersAndSetters(const OpSpec& op, SourceWriter* writer) {
for (const AttributeSpec& attr : op.optional_attributes()) {
Method setter = Method::Create(attr.var().name(), Type::Class("Options"));
Javadoc setter_doc = Javadoc::Create();
AddArgument(attr.var(), attr.description(), &setter, &setter_doc);
writer->BeginMethod(setter, PUBLIC | STATIC, &setter_doc)
.Append("return new Options()." + attr.var().name() + "(" +
attr.var().name() + ");")
.EndLine()
.EndMethod();
}
for (const ArgumentSpec& output : op.outputs()) {
Method getter = Method::Create(output.var().name(), output.var().type());
Javadoc getter_doc = Javadoc::Create(output.description());
writer->BeginMethod(getter, PUBLIC, &getter_doc)
.Append("return " + output.var().name() + ";")
.EndLine()
.EndMethod();
}
}
void RenderInterfaceImpl(const OpSpec& op, RenderMode mode,
SourceWriter* writer) {
ArgumentSpec output = op.outputs().front();
if (mode == OPERAND) {
bool cast2obj = output.type().wildcard();
Type return_type =
Type::Class("Output", "org.tensorflow")
.add_parameter(cast2obj ? Type::Class("Object") : output.type());
Method as_output = Method::Create("asOutput", return_type)
.add_annotation(Annotation::Create("Override"));
if (cast2obj) {
as_output.add_annotation(
Annotation::Create("SuppressWarnings").attributes("\"unchecked\""));
}
writer->BeginMethod(as_output, PUBLIC);
if (cast2obj) {
writer->Append("return (").AppendType(return_type).Append(") ");
} else {
writer->Append("return ");
}
writer->Append(output.var().name() + ";").EndLine().EndMethod();
} else if (mode == LIST_OPERAND) {
Type operand = Type::Interface("Operand", "org.tensorflow");
if (output.type().wildcard()) {
operand.add_parameter(Type::Class("Object"));
} else {
operand.add_parameter(output.type());
}
Type return_type =
Type::Interface("Iterator", "java.util").add_parameter(operand);
Method iterator =
Method::Create("iterator", return_type)
.add_annotation(Annotation::Create("Override"))
.add_annotation(Annotation::Create("SuppressWarnings")
.attributes("{\"rawtypes\", \"unchecked\"}"));
// cast the output list using a raw List
writer->BeginMethod(iterator, PUBLIC)
.Append("return (" + return_type.name() + ") ")
.Append(output.var().name() + ".iterator();")
.EndLine()
.EndMethod();
}
}
void RenderOptionsClass(const OpSpec& op, const Type& op_class,
SourceWriter* writer) {
Type options_class = Type::Class("Options");
Javadoc options_doc = Javadoc::Create("Optional attributes for {@link " +
op_class.canonical_name() + "}");
writer->BeginInnerType(options_class, PUBLIC | STATIC, &options_doc);
for (const AttributeSpec& attr : op.optional_attributes()) {
Method setter = Method::Create(attr.var().name(), options_class);
Javadoc setter_doc = Javadoc::Create();
AddArgument(attr.var(), attr.description(), &setter, &setter_doc);
writer->BeginMethod(setter, PUBLIC, &setter_doc)
.Append("this." + attr.var().name() + " = " + attr.var().name() + ";")
.EndLine()
.Append("return this;")
.EndLine()
.EndMethod();
}
writer->EndLine();
for (const AttributeSpec& optional_attribute : op.optional_attributes()) {
writer->WriteField(optional_attribute.var(), PRIVATE);
}
Method constructor = Method::ConstructorFor(options_class);
writer->BeginMethod(constructor, PRIVATE).EndMethod();
writer->EndType();
}
inline Type ClassOf(const EndpointSpec& endpoint,
const std::string& base_package) {
return Type::Class(
endpoint.name(),
base_package + "." + absl::AsciiStrToLower(endpoint.package()));
}
void GenerateOp(const OpSpec& op, const EndpointSpec& endpoint,
const std::string& base_package, const std::string& output_dir,
Env* env) {
Type op_class(
ClassOf(endpoint, base_package)
.add_supertype(Type::Class("PrimitiveOp", "org.tensorflow.op")));
Javadoc op_javadoc(endpoint.javadoc());
// op interfaces
RenderMode mode = DEFAULT;
if (op.outputs().size() == 1) {
const ArgumentSpec& output = op.outputs().front();
Type operand_type(output.type().wildcard() ? Type::Class("Object")
: output.type());
Type operand_inf(Type::Interface("Operand", "org.tensorflow")
.add_parameter(operand_type));
if (output.iterable()) {
mode = LIST_OPERAND;
op_class.add_supertype(Type::IterableOf(operand_inf));
} else {
mode = OPERAND;
op_class.add_supertype(operand_inf);
}
}
// op generic parameters
std::set<std::string> generics;
for (const ArgumentSpec& output : op.outputs()) {
if (output.type().kind() == Type::GENERIC && !output.type().wildcard() &&
generics.find(output.type().name()) == generics.end()) {
op_class.add_parameter(output.type());
op_javadoc.add_param_tag(
"<" + output.type().name() + ">",
"data type for {@code " + output.var().name() + "()} output");
generics.insert(output.type().name());
}
}
// op annotations
if (endpoint.deprecated()) {
op_class.add_annotation(Annotation::Create("Deprecated"));
std::string explanation;
if (!op.endpoints().front().deprecated()) {
explanation =
"use {@link " +
ClassOf(op.endpoints().front(), base_package).canonical_name() +
"} instead";
} else {
explanation = op.deprecation_explanation();
}
op_javadoc.add_tag("deprecated", explanation);
}
if (!op.hidden()) {
// expose the op in the Ops Graph API only if it is visible
Annotation oper_annot =
Annotation::Create("Operator", "org.tensorflow.op.annotation");
if (endpoint.package() != kDefaultEndpointPackage) {
oper_annot.attributes("group = \"" + endpoint.package() + "\"");
}
op_class.add_annotation(oper_annot);
}
// create op class file
const std::string op_dir_name = io::JoinPath(
output_dir, str_util::StringReplace(op_class.package(), ".", "/", true));
if (!env->FileExists(op_dir_name).ok()) {
TF_CHECK_OK(Env::Default()->RecursivelyCreateDir(op_dir_name))
<< op_dir_name;
}
const std::string op_file_name = op_class.name() + ".java";
std::unique_ptr<tensorflow::WritableFile> op_file;
TF_CHECK_OK(
env->NewWritableFile(io::JoinPath(op_dir_name, op_file_name), &op_file))
<< op_file_name;
// render endpoint source code
SourceFileWriter writer(op_file.get());
std::list<Type> dependencies;
CollectOpDependencies(op, mode, &dependencies);
writer.Write(kLicense)
.EndLine()
.Write("// This class has been generated, DO NOT EDIT!")
.EndLine()
.EndLine()
.BeginType(op_class, PUBLIC | FINAL, &dependencies, &op_javadoc);
if (!op.optional_attributes().empty()) {
RenderOptionsClass(op, op_class, &writer);
}
RenderFactoryMethods(op, op_class, &writer);
RenderGettersAndSetters(op, &writer);
if (mode != DEFAULT) {
RenderInterfaceImpl(op, mode, &writer);
}
writer.EndLine();
for (const ArgumentSpec& output : op.outputs()) {
writer.WriteField(output.var(), PRIVATE);
}
RenderConstructor(op, op_class, &writer);
writer.EndType();
}
bool CanGenerateOp(const OpDef& op_def, const ApiDef& api_def) {
if (api_def.visibility() == ApiDef::SKIP) {
return false;
}
for (const auto& attr : op_def.attr()) {
if (attr.type() == "func" || attr.type() == "list(func)") {
return false; // TODO(karllessard) add support for function attributes
}
}
return true;
}
} // namespace
absl::Status OpGenerator::Run(const OpList& op_list,
const std::string& base_package,
const std::string& output_dir) {
ApiDefMap api_map(op_list);
if (!api_dirs_.empty()) {
// Only load api files that correspond to the requested "op_list"
for (const auto& op : op_list.op()) {
for (const auto& api_def_dir : api_dirs_) {
const std::string api_def_file_pattern =
io::JoinPath(api_def_dir, "api_def_" + op.name() + ".pbtxt");
if (env_->FileExists(api_def_file_pattern).ok()) {
TF_CHECK_OK(api_map.LoadFile(env_, api_def_file_pattern))
<< api_def_file_pattern;
}
}
}
}
api_map.UpdateDocs();
for (const auto& op_def : op_list.op()) {
const ApiDef* api_def = api_map.GetApiDef(op_def.name());
if (CanGenerateOp(op_def, *api_def)) {
OpSpec op(OpSpec::Create(op_def, *api_def));
for (const EndpointSpec& endpoint : op.endpoints()) {
GenerateOp(op, endpoint, base_package, output_dir, env_);
}
}
}
return absl::OkStatus();
}
} // namespace java
} // namespace tensorflow
tensorflow/java/src/gen/cc/op_generator.h
+60
View File
@@ -0,0 +1,60 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_GEN_CC_OP_GENERATOR_H_
#define TENSORFLOW_JAVA_SRC_GEN_CC_OP_GENERATOR_H_
#include <string>
#include <vector>
#include "absl/status/status.h"
#include "tensorflow/core/framework/api_def.pb.h"
#include "tensorflow/core/framework/op_def.pb.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/java/src/gen/cc/op_specs.h"
namespace tensorflow {
namespace java {
// A generator of Java operation wrappers.
//
// This generator takes a list of ops definitions in input and outputs
// a Java Op wrapper for each of them in the provided directory. The same
// generator instance can be invoked multiple times with a different list of
// ops definitions.
class OpGenerator {
public:
explicit OpGenerator(const std::vector<std::string>& api_dirs,
Env* env = Env::Default())
: api_dirs_(api_dirs), env_(env) {}
// Generates wrappers for the given list of 'ops'.
//
// Output files are generated in <output_dir>/<base_package>/<op_package>,
// where 'op_package' is derived from ops endpoints.
absl::Status Run(const OpList& op_list, const std::string& base_package,
const std::string& output_dir);
private:
const std::vector<std::string> api_dirs_;
Env* env_;
};
} // namespace java
} // namespace tensorflow
#endif // TENSORFLOW_JAVA_SRC_GEN_CC_OP_GENERATOR_H_
tensorflow/java/src/gen/cc/op_specs.cc
+405
View File
@@ -0,0 +1,405 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/java/src/gen/cc/op_specs.h"
#include <map>
#include <sstream>
#include <string>
#include <utility>
#include <vector>
#include "absl/log/log.h"
#include "absl/strings/ascii.h"
#include "absl/strings/match.h"
#include "absl/strings/str_join.h"
#include "absl/strings/strip.h"
#include "re2/re2.h"
#include "tensorflow/core/framework/api_def.pb.h"
#include "tensorflow/core/framework/attr_value.pb.h"
#include "tensorflow/core/framework/op_def.pb.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/stringpiece.h"
#include "tensorflow/java/src/gen/cc/java_defs.h"
namespace tensorflow {
namespace java {
namespace {
inline bool IsRealNumbers(const AttrValue& values) {
if (!values.has_list()) {
return RealNumberTypes().Contains(values.type());
}
for (int i = 0; i < values.list().type_size(); ++i) {
if (!RealNumberTypes().Contains(values.list().type(i))) {
return false;
}
}
return true;
}
class TypeResolver {
public:
explicit TypeResolver(const OpDef& op_def) : op_def_(op_def) {}
// Returns the class type of an input/output argument
//
// For example, if the argument's datatype is DT_STRING, this method will
// return "java.lang.String", so the argument can become "Operand<String>"
// in the Ops API
Type TypeOf(const OpDef_ArgDef& arg_def, bool* iterable_out);
// Returns types of an input attribute
//
// The first element of the pair is the class type of this attribute while
// the second is its JNI/primitive type equivalent, required for explicit
// unboxing.
//
// For example, if the attribute is of type "float", this method will return
// <java.lang.Float, float>, so the attribute can be used as a "Float" object
// in the Ops API and casted to a "float" when passing through the JNI layer.
std::pair<Type, Type> TypesOf(const OpDef_AttrDef& attr_def,
bool* iterable_out);
// Returns true if the type of this attribute has already been resolved
bool IsAttributeVisited(const std::string& attr_name) {
return visited_attrs_.find(attr_name) != visited_attrs_.cend();
}
private:
const OpDef op_def_;
std::map<std::string, Type> visited_attrs_;
char next_generic_letter_ = 'T';
std::pair<Type, Type> MakeTypePair(const Type& type, const Type& jni_type) {
return std::make_pair(type, jni_type);
}
std::pair<Type, Type> MakeTypePair(const Type& type) {
return std::make_pair(type, type);
}
Type NextGeneric() {
char generic_letter = next_generic_letter_++;
if (next_generic_letter_ > 'Z') {
next_generic_letter_ = 'A';
}
return Type::Generic(std::string(1, generic_letter));
}
};
Type TypeResolver::TypeOf(const OpDef_ArgDef& arg_def, bool* iterable_out) {
*iterable_out = false;
Type type = Type::Wildcard();
if (arg_def.type() != DataType::DT_INVALID) {
type = Type::ForDataType(arg_def.type());
} else if (!arg_def.type_attr().empty()) {
// resolve type from attribute (if already visited, retrieve its type)
if (IsAttributeVisited(arg_def.type_attr())) {
type = visited_attrs_.at(arg_def.type_attr());
} else {
for (const auto& attr_def : op_def_.attr()) {
if (attr_def.name() == arg_def.type_attr()) {
type = TypesOf(attr_def, iterable_out).first;
break;
}
}
}
} else if (!arg_def.type_list_attr().empty()) {
// type is a list of tensors that can be of different data types, so leave
// it as a list of wildcards
*iterable_out = true;
visited_attrs_.insert(std::make_pair(arg_def.type_list_attr(), type));
} else {
LOG(FATAL) << "Cannot resolve data type of argument \"" << arg_def.name()
<< "\" in operation \"" << op_def_.name() << "\"";
}
if (!arg_def.number_attr().empty()) {
// when number_attr is set, argument has to be a list of tensors
*iterable_out = true;
visited_attrs_.insert(std::make_pair(arg_def.number_attr(), Type::Int()));
}
return type;
}
std::pair<Type, Type> TypeResolver::TypesOf(const OpDef_AttrDef& attr_def,
bool* iterable_out) {
std::pair<Type, Type> types = MakeTypePair(Type::Wildcard());
*iterable_out = false;
absl::string_view attr_type = attr_def.type();
if (absl::ConsumePrefix(&attr_type, "list(")) {
attr_type.remove_suffix(1); // remove closing brace
*iterable_out = true;
}
if (attr_type == "string") {
types = MakeTypePair(Type::Class("String"));
} else if (attr_type == "int") {
types = MakeTypePair(Type::Class("Long"), Type::Long());
} else if (attr_type == "float") {
types = MakeTypePair(Type::Class("Float"), Type::Float());
} else if (attr_type == "bool") {
types = MakeTypePair(Type::Class("Boolean"), Type::Boolean());
} else if (attr_type == "shape") {
types = MakeTypePair(Type::Class("Shape", "org.tensorflow"));
} else if (attr_type == "tensor") {
types = MakeTypePair(Type::Class("Tensor", "org.tensorflow")
.add_parameter(Type::Wildcard()));
} else if (attr_type == "type") {
Type type = *iterable_out ? Type::Wildcard() : NextGeneric();
if (IsRealNumbers(attr_def.allowed_values())) {
type.add_supertype(Type::Class("Number"));
}
types = MakeTypePair(type, Type::Enum("DataType", "org.tensorflow"));
} else {
LOG(FATAL) << "Cannot resolve data type for attribute \"" << attr_type
<< "\" in operation \"" << op_def_.name() << "\"";
}
visited_attrs_.insert(std::make_pair(attr_def.name(), types.first));
return types;
}
std::string SnakeToCamelCase(const std::string& str, bool upper = false) {
std::string result;
bool cap = upper;
for (std::string::const_iterator it = str.begin(); it != str.end(); ++it) {
const char c = *it;
if (c == '_') {
cap = true;
} else if (cap) {
result += absl::ascii_toupper(c);
cap = false;
} else {
result += c;
}
}
return result;
}
bool FindAndCut(std::string* input, const RE2& expr, std::string* before_match,
std::string* ret_match = nullptr) {
std::string match;
if (!RE2::PartialMatch(*input, expr, &match)) return false;
*before_match = input->substr(0, input->find(match));
*input = input->substr(before_match->size() + match.size());
if (ret_match != nullptr) *ret_match = match;
return true;
}
std::string ParseDocumentation(const std::string& inp) {
std::stringstream javadoc_text;
// TODO(karllessard) This is a very minimalist utility method for converting
// markdown syntax, as found in ops descriptions, to Javadoc/html tags. Check
// for alternatives to increase the level of support for markups.
std::vector<std::string> markups_subexpr;
markups_subexpr.push_back("\n+\\*\\s+"); // lists
markups_subexpr.push_back("\n{2,}"); // paragraphs
markups_subexpr.push_back("`{3,}\\s*[^\\s\n]*\\s*\n"); // code blocks
markups_subexpr.push_back("`+"); // inlined code and code blocks
markups_subexpr.push_back("\\*{1,2}\\b"); // text emphasis
markups_subexpr.push_back("\\["); // hyperlinks
const RE2 markup_expr("(" + absl::StrJoin(markups_subexpr, "|") + ")");
bool in_list = false;
std::string input = inp;
while (true) {
std::string text, markup;
if (!FindAndCut(&input, markup_expr, &text, &markup)) {
javadoc_text << input;
break; // end of loop
}
javadoc_text << text;
if (absl::StartsWith(markup, "\n")) {
javadoc_text << "\n";
if (absl::StrContains(markup, "*")) {
// new list item
javadoc_text << (in_list ? "</li>\n" : "<ul>\n") << "<li>\n";
in_list = true;
} else if (in_list) {
// end of list
javadoc_text << "</li>\n</ul>\n";
in_list = false;
} else if (!absl::StartsWith(input, "```")) {
// new paragraph (not required if a <pre> block follows)
javadoc_text << "<p>\n";
}
} else if (absl::StartsWith(markup, "```")) {
// code blocks
if (FindAndCut(&input, "(```\\s*\n*)", &text)) {
javadoc_text << "<pre>{@code\n" << text << "}</pre>\n";
} else {
javadoc_text << markup;
}
} else if (absl::StartsWith("(" + markup + ")", "`")) {
// inlined code
if (FindAndCut(&input, markup, &text)) {
javadoc_text << "{@code " << text << "}";
} else {
javadoc_text << markup;
}
} else if (markup == "**") {
// text emphasis (strong)
if (FindAndCut(&input, "(\\b\\*{2})", &text)) {
javadoc_text << "<b>" << ParseDocumentation(text) << "</b>";
} else {
javadoc_text << markup;
}
} else if (markup == "*") {
// text emphasis (normal)
if (FindAndCut(&input, "(\\b\\*{1})", &text)) {
javadoc_text << "<i>" << ParseDocumentation(text) << "</i>";
} else {
javadoc_text << markup;
}
} else if (absl::StartsWith(markup, "[")) {
// hyperlinks
std::string label;
std::string link;
if (RE2::PartialMatch(input, "([^\\[]+)\\]\\((http.+)\\)", &label,
&link) &&
absl::StartsWith(input, label + link)) {
input = input.substr(label.size() + link.size());
javadoc_text << "<a href=\"" << link << "\">"
<< ParseDocumentation(label) << "</a>";
} else {
javadoc_text << markup;
}
} else {
// safe fallback
javadoc_text << markup;
}
}
return javadoc_text.str();
}
ArgumentSpec CreateInput(const OpDef_ArgDef& input_def,
const ApiDef::Arg& input_api_def,
TypeResolver* type_resolver) {
bool iterable = false;
Type type = type_resolver->TypeOf(input_def, &iterable);
Type var_type =
Type::Interface("Operand", "org.tensorflow").add_parameter(type);
if (iterable) {
var_type = Type::IterableOf(var_type);
}
return ArgumentSpec(
input_api_def.name(),
Variable::Create(SnakeToCamelCase(input_api_def.rename_to()), var_type),
type, ParseDocumentation(input_api_def.description()), iterable);
}
AttributeSpec CreateAttribute(const OpDef_AttrDef& attr_def,
const ApiDef::Attr& attr_api_def,
TypeResolver* type_resolver) {
bool iterable = false;
std::pair<Type, Type> types = type_resolver->TypesOf(attr_def, &iterable);
Type var_type = types.first.kind() == Type::GENERIC
? Type::ClassOf(types.first)
: types.first;
if (iterable) {
var_type = Type::ListOf(var_type);
}
return AttributeSpec(
attr_api_def.name(),
Variable::Create(SnakeToCamelCase(attr_api_def.rename_to()), var_type),
types.first, types.second, ParseDocumentation(attr_api_def.description()),
iterable,
attr_def.has_default_value() ? &attr_def.default_value() : nullptr);
}
ArgumentSpec CreateOutput(const OpDef_ArgDef& output_def,
const ApiDef::Arg& output_api,
TypeResolver* type_resolver) {
bool iterable = false;
Type type = type_resolver->TypeOf(output_def, &iterable);
Type var_type = Type::Class("Output", "org.tensorflow").add_parameter(type);
if (iterable) {
var_type = Type::ListOf(var_type);
}
return ArgumentSpec(
output_api.name(),
Variable::Create(SnakeToCamelCase(output_api.rename_to()), var_type),
type, ParseDocumentation(output_api.description()), iterable);
}
EndpointSpec CreateEndpoint(const OpDef& op_def, const ApiDef& api_def,
const ApiDef_Endpoint& endpoint_def) {
std::vector<std::string> name_tokens =
str_util::Split(endpoint_def.name(), ".");
std::string package;
std::string name;
if (name_tokens.size() > 1) {
package = name_tokens.at(0);
name = name_tokens.at(1);
} else {
package = "core"; // generate unclassified ops in the 'core' package
name = name_tokens.at(0);
}
return EndpointSpec(package, name,
Javadoc::Create(ParseDocumentation(api_def.summary()))
.details(ParseDocumentation(api_def.description())));
}
} // namespace
OpSpec OpSpec::Create(const OpDef& op_def, const ApiDef& api_def) {
OpSpec op(api_def.graph_op_name(), api_def.visibility() == ApiDef::HIDDEN,
op_def.deprecation().explanation());
TypeResolver type_resolver(op_def);
for (const std::string& next_input_name : api_def.arg_order()) {
for (int i = 0; i < op_def.input_arg().size(); ++i) {
if (op_def.input_arg(i).name() == next_input_name) {
op.inputs_.push_back(CreateInput(op_def.input_arg(i), api_def.in_arg(i),
&type_resolver));
break;
}
}
}
for (int i = 0; i < op_def.attr().size(); ++i) {
// do not parse attributes already visited, they have probably been inferred
// before as an input argument type
if (!type_resolver.IsAttributeVisited(op_def.attr(i).name())) {
AttributeSpec attr =
CreateAttribute(op_def.attr(i), api_def.attr(i), &type_resolver);
// attributes with a default value are optional
if (attr.has_default_value() && attr.type().kind() != Type::GENERIC) {
op.optional_attributes_.push_back(attr);
} else {
op.attributes_.push_back(attr);
}
}
}
for (int i = 0; i < op_def.output_arg().size(); ++i) {
op.outputs_.push_back(
CreateOutput(op_def.output_arg(i), api_def.out_arg(i), &type_resolver));
}
for (const auto& endpoint_def : api_def.endpoint()) {
op.endpoints_.push_back(CreateEndpoint(op_def, api_def, endpoint_def));
}
return op;
}
} // namespace java
} // namespace tensorflow
tensorflow/java/src/gen/cc/op_specs.h
+180
View File
@@ -0,0 +1,180 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_GEN_CC_OP_SPECS_H_
#define TENSORFLOW_JAVA_SRC_GEN_CC_OP_SPECS_H_
#include <string>
#include <vector>
#include "tensorflow/core/framework/api_def.pb.h"
#include "tensorflow/core/framework/attr_value.pb.h"
#include "tensorflow/core/framework/op_def.pb.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/java/src/gen/cc/java_defs.h"
namespace tensorflow {
namespace java {
constexpr const char kDefaultEndpointPackage[] = "core";
class EndpointSpec {
public:
// A specification for an operation endpoint
//
// package: package of this endpoint (from which also derives its package)
// name: name of this endpoint class
// javadoc: the endpoint class documentation
// TODO(annarev): hardcode deprecated to false until deprecated is possible
EndpointSpec(const std::string& package, const std::string& name,
const Javadoc& javadoc)
: package_(package), name_(name), javadoc_(javadoc), deprecated_(false) {}
const std::string& package() const { return package_; }
const std::string& name() const { return name_; }
const Javadoc& javadoc() const { return javadoc_; }
bool deprecated() const { return deprecated_; }
private:
const std::string package_;
const std::string name_;
const Javadoc javadoc_;
const bool deprecated_;
};
class ArgumentSpec {
public:
// A specification for an operation argument
//
// op_def_name: argument name, as known by TensorFlow core
// var: a variable to represent this argument in Java
// type: the tensor type of this argument
// description: a description of this argument, in javadoc
// iterable: true if this argument is a list
ArgumentSpec(const std::string& op_def_name, const Variable& var,
const Type& type, const std::string& description, bool iterable)
: op_def_name_(op_def_name),
var_(var),
type_(type),
description_(description),
iterable_(iterable) {}
const std::string& op_def_name() const { return op_def_name_; }
const Variable& var() const { return var_; }
const Type& type() const { return type_; }
const std::string& description() const { return description_; }
bool iterable() const { return iterable_; }
private:
const std::string op_def_name_;
const Variable var_;
const Type type_;
const std::string description_;
const bool iterable_;
};
class AttributeSpec {
public:
// A specification for an operation attribute
//
// op_def_name: attribute name, as known by TensorFlow core
// var: a variable to represent this attribute in Java
// type: the type of this attribute
// jni_type: the type of this attribute in JNI layer (see OperationBuilder)
// description: a description of this attribute, in javadoc
// iterable: true if this attribute is a list
// default_value: default value for this attribute or nullptr if none. Any
// value referenced by this pointer must outlive the lifetime
// of the AttributeSpec. This is guaranteed if the value is
// issued by an OpDef of the global OpRegistry.
AttributeSpec(const std::string& op_def_name, const Variable& var,
const Type& type, const Type& jni_type,
const std::string& description, bool iterable,
const AttrValue* default_value)
: op_def_name_(op_def_name),
var_(var),
type_(type),
description_(description),
iterable_(iterable),
jni_type_(jni_type),
default_value_(default_value) {}
const std::string& op_def_name() const { return op_def_name_; }
const Variable& var() const { return var_; }
const Type& type() const { return type_; }
const std::string& description() const { return description_; }
bool iterable() const { return iterable_; }
const Type& jni_type() const { return jni_type_; }
bool has_default_value() const { return default_value_ != nullptr; }
const AttrValue* default_value() const { return default_value_; }
private:
const std::string op_def_name_;
const Variable var_;
const Type type_;
const std::string description_;
const bool iterable_;
const Type jni_type_;
const AttrValue* default_value_;
};
class OpSpec {
public:
// Parses an op definition and its API to produce a specification used for
// rendering its Java wrapper
//
// op_def: Op definition
// api_def: Op API definition
static OpSpec Create(const OpDef& op_def, const ApiDef& api_def);
const std::string& graph_op_name() const { return graph_op_name_; }
bool hidden() const { return hidden_; }
const std::string& deprecation_explanation() const {
return deprecation_explanation_;
}
const std::vector<EndpointSpec> endpoints() const { return endpoints_; }
const std::vector<ArgumentSpec>& inputs() const { return inputs_; }
const std::vector<ArgumentSpec>& outputs() const { return outputs_; }
const std::vector<AttributeSpec>& attributes() const { return attributes_; }
const std::vector<AttributeSpec>& optional_attributes() const {
return optional_attributes_;
}
private:
// A specification for an operation
//
// graph_op_name: name of this op, as known by TensorFlow core engine
// hidden: true if this op should not be visible through the Graph Ops API
// deprecation_explanation: message to show if all endpoints are deprecated
explicit OpSpec(const std::string& graph_op_name, bool hidden,
const std::string& deprecation_explanation)
: graph_op_name_(graph_op_name),
hidden_(hidden),
deprecation_explanation_(deprecation_explanation) {}
const std::string graph_op_name_;
const bool hidden_;
const std::string deprecation_explanation_;
std::vector<EndpointSpec> endpoints_;
std::vector<ArgumentSpec> inputs_;
std::vector<ArgumentSpec> outputs_;
std::vector<AttributeSpec> attributes_;
std::vector<AttributeSpec> optional_attributes_;
};
} // namespace java
} // namespace tensorflow
#endif // TENSORFLOW_JAVA_SRC_GEN_CC_OP_SPECS_H_
tensorflow/java/src/gen/cc/source_writer.cc
+371
View File
@@ -0,0 +1,371 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/java/src/gen/cc/source_writer.h"
#include <algorithm>
#include <cstddef>
#include <list>
#include <string>
#include "absl/log/check.h"
#include "xla/tsl/platform/status.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/stringpiece.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/java/src/gen/cc/java_defs.h"
namespace tensorflow {
namespace java {
SourceWriter::SourceWriter() {
// Push an empty generic namespace at start, for simplification.
generic_namespaces_.push(new GenericNamespace());
}
SourceWriter::~SourceWriter() {
// Remove empty generic namespace added at start as well as any other
// namespace objects that haven't been removed.
while (!generic_namespaces_.empty()) {
GenericNamespace* generic_namespace = generic_namespaces_.top();
generic_namespaces_.pop();
delete generic_namespace;
}
}
SourceWriter& SourceWriter::Indent(int tab) {
left_margin_.resize(
std::max(static_cast<int>(left_margin_.size() + tab), 0), ' ');
return *this;
}
SourceWriter& SourceWriter::Prefix(const char* line_prefix) {
line_prefix_ = line_prefix;
return *this;
}
SourceWriter& SourceWriter::Write(const absl::string_view& str) {
size_t line_pos = 0;
do {
size_t start_pos = line_pos;
line_pos = str.find('\n', start_pos);
if (line_pos != std::string::npos) {
++line_pos;
Append(str.substr(start_pos, line_pos - start_pos));
newline_ = true;
} else {
Append(str.substr(start_pos, str.size() - start_pos));
}
} while (line_pos != std::string::npos && line_pos < str.size());
return *this;
}
SourceWriter& SourceWriter::WriteFromFile(const std::string& fname, Env* env) {
std::string data_;
TF_CHECK_OK(ReadFileToString(env, fname, &data_));
return Write(data_);
}
SourceWriter& SourceWriter::Append(const absl::string_view& str) {
if (!str.empty()) {
if (newline_) {
DoAppend(left_margin_ + line_prefix_);
newline_ = false;
}
DoAppend(str);
}
return *this;
}
SourceWriter& SourceWriter::AppendType(const Type& type) {
if (type.wildcard()) {
Append("?");
} else {
Append(type.name());
if (!type.parameters().empty()) {
Append("<");
bool first = true;
for (const Type& t : type.parameters()) {
if (!first) {
Append(", ");
}
AppendType(t);
first = false;
}
Append(">");
}
}
return *this;
}
SourceWriter& SourceWriter::EndLine() {
Append("\n");
newline_ = true;
return *this;
}
SourceWriter& SourceWriter::BeginBlock(const std::string& expression) {
if (!expression.empty()) {
Append(expression + " {");
} else {
Append(newline_ ? "{" : " {");
}
return EndLine().Indent(2);
}
SourceWriter& SourceWriter::EndBlock() {
return Indent(-2).Append("}").EndLine();
}
SourceWriter& SourceWriter::BeginMethod(const Method& method, int modifiers,
const Javadoc* javadoc) {
GenericNamespace* generic_namespace = PushGenericNamespace(modifiers);
if (!method.constructor()) {
generic_namespace->Visit(method.return_type());
}
for (const Variable& v : method.arguments()) {
generic_namespace->Visit(v.type());
}
EndLine();
if (javadoc != nullptr) {
WriteJavadoc(*javadoc);
}
if (!method.annotations().empty()) {
WriteAnnotations(method.annotations());
}
WriteModifiers(modifiers);
if (!generic_namespace->declared_types().empty()) {
WriteGenerics(generic_namespace->declared_types());
Append(" ");
}
if (!method.constructor()) {
AppendType(method.return_type()).Append(" ");
}
Append(method.name()).Append("(");
bool first = true;
for (const Variable& v : method.arguments()) {
if (!first) {
Append(", ");
}
AppendType(v.type()).Append(v.variadic() ? "... " : " ").Append(v.name());
first = false;
}
return Append(")").BeginBlock();
}
SourceWriter& SourceWriter::EndMethod() {
EndBlock();
PopGenericNamespace();
return *this;
}
SourceWriter& SourceWriter::BeginType(const Type& type, int modifiers,
const std::list<Type>* extra_dependencies,
const Javadoc* javadoc) {
if (!type.package().empty()) {
Append("package ").Append(type.package()).Append(";").EndLine();
}
TypeImporter type_importer(type.package());
type_importer.Visit(type);
if (extra_dependencies != nullptr) {
for (const Type& t : *extra_dependencies) {
type_importer.Visit(t);
}
}
if (!type_importer.imports().empty()) {
EndLine();
for (const std::string& s : type_importer.imports()) {
Append("import ").Append(s).Append(";").EndLine();
}
}
return BeginInnerType(type, modifiers, javadoc);
}
SourceWriter& SourceWriter::BeginInnerType(const Type& type, int modifiers,
const Javadoc* javadoc) {
GenericNamespace* generic_namespace = PushGenericNamespace(modifiers);
generic_namespace->Visit(type);
EndLine();
if (javadoc != nullptr) {
WriteJavadoc(*javadoc);
}
if (!type.annotations().empty()) {
WriteAnnotations(type.annotations());
}
WriteModifiers(modifiers);
CHECK_EQ(Type::Kind::CLASS, type.kind()) << ": Not supported yet";
Append("class ").Append(type.name());
if (!generic_namespace->declared_types().empty()) {
WriteGenerics(generic_namespace->declared_types());
}
if (!type.supertypes().empty()) {
bool first_interface = true;
for (const Type& t : type.supertypes()) {
if (t.kind() == Type::CLASS) { // superclass is always first in list
Append(" extends ");
} else if (first_interface) {
Append(" implements ");
first_interface = false;
} else {
Append(", ");
}
AppendType(t);
}
}
return BeginBlock();
}
SourceWriter& SourceWriter::EndType() {
EndBlock();
PopGenericNamespace();
return *this;
}
SourceWriter& SourceWriter::WriteField(const Variable& field, int modifiers,
const Javadoc* javadoc) {
// If present, write field javadoc only as one brief line
if (javadoc != nullptr && !javadoc->brief().empty()) {
Append("/** ").Append(javadoc->brief()).Append(" */").EndLine();
}
WriteModifiers(modifiers);
AppendType(field.type()).Append(" ").Append(field.name()).Append(";");
EndLine();
return *this;
}
SourceWriter& SourceWriter::WriteModifiers(int modifiers) {
if (modifiers & PUBLIC) {
Append("public ");
} else if (modifiers & PROTECTED) {
Append("protected ");
} else if (modifiers & PRIVATE) {
Append("private ");
}
if (modifiers & STATIC) {
Append("static ");
}
if (modifiers & FINAL) {
Append("final ");
}
return *this;
}
SourceWriter& SourceWriter::WriteJavadoc(const Javadoc& javadoc) {
Append("/**").Prefix(" * ").EndLine();
bool do_line_break = false;
if (!javadoc.brief().empty()) {
Write(javadoc.brief()).EndLine();
do_line_break = true;
}
if (!javadoc.details().empty()) {
if (do_line_break) {
Append("<p>").EndLine();
}
Write(javadoc.details()).EndLine();
do_line_break = true;
}
if (!javadoc.tags().empty()) {
if (do_line_break) {
EndLine();
}
for (const auto& p : javadoc.tags()) {
Append("@" + p.first);
if (!p.second.empty()) {
Append(" ").Write(p.second);
}
EndLine();
}
}
return Prefix("").Append(" */").EndLine();
}
SourceWriter& SourceWriter::WriteAnnotations(
const std::list<Annotation>& annotations) {
for (const Annotation& a : annotations) {
Append("@" + a.name());
if (!a.attributes().empty()) {
Append("(").Append(a.attributes()).Append(")");
}
EndLine();
}
return *this;
}
SourceWriter& SourceWriter::WriteGenerics(
const std::list<const Type*>& generics) {
Append("<");
bool first = true;
for (const Type* pt : generics) {
if (!first) {
Append(", ");
}
Append(pt->name());
if (!pt->supertypes().empty()) {
Append(" extends ").AppendType(pt->supertypes().front());
}
first = false;
}
return Append(">");
}
SourceWriter::GenericNamespace* SourceWriter::PushGenericNamespace(
int modifiers) {
GenericNamespace* generic_namespace;
if (modifiers & STATIC) {
generic_namespace = new GenericNamespace();
} else {
generic_namespace = new GenericNamespace(generic_namespaces_.top());
}
generic_namespaces_.push(generic_namespace);
return generic_namespace;
}
void SourceWriter::PopGenericNamespace() {
GenericNamespace* generic_namespace = generic_namespaces_.top();
generic_namespaces_.pop();
delete generic_namespace;
}
void SourceWriter::TypeVisitor::Visit(const Type& type) {
DoVisit(type);
for (const Type& t : type.parameters()) {
Visit(t);
}
for (const Annotation& t : type.annotations()) {
DoVisit(t);
}
for (const Type& t : type.supertypes()) {
Visit(t);
}
}
void SourceWriter::GenericNamespace::DoVisit(const Type& type) {
// ignore non-generic parameters, wildcards and generics already declared
if (type.kind() == Type::GENERIC && !type.wildcard() &&
generic_names_.find(type.name()) == generic_names_.end()) {
declared_types_.push_back(&type);
generic_names_.insert(type.name());
}
}
void SourceWriter::TypeImporter::DoVisit(const Type& type) {
if (!type.package().empty() && type.package() != current_package_) {
imports_.insert(type.canonical_name());
}
}
} // namespace java
} // namespace tensorflow
tensorflow/java/src/gen/cc/source_writer.h
+261
View File
@@ -0,0 +1,261 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_GEN_CC_SOURCE_WRITER_H_
#define TENSORFLOW_JAVA_SRC_GEN_CC_SOURCE_WRITER_H_
#include <list>
#include <set>
#include <stack>
#include <string>
#include "xla/tsl/platform/status.h"
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/file_system.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/java/src/gen/cc/java_defs.h"
namespace tensorflow {
namespace java {
// A class for writing Java source code.
class SourceWriter {
public:
SourceWriter();
virtual ~SourceWriter();
// Indents following lines with white spaces.
//
// Indentation is cumulative, i.e. the provided tabulation is added to the
// current indentation value. If the tabulation is negative, the operation
// will outdent the source code, until the indentation reaches 0 again.
//
// For example, calling Indent(2) twice will indent code with 4 white
// spaces. Then calling Indent(-2) will outdent the code back to 2 white
// spaces.
SourceWriter& Indent(int tab);
// Prefixes following lines with provided character(s).
//
// A common use case of a prefix is for commenting or documenting the code.
//
// The prefix is written after the indentation, For example, invoking
// Indent(2)->Prefix("//") will result in prefixing lines with " //".
//
// An empty value ("") will remove any line prefix that was previously set.
SourceWriter& Prefix(const char* line_prefix);
// Writes a source code snippet.
//
// The data might potentially contain newline characters, therefore it will
// be scanned to ensure that each line is indented and prefixed properly,
// making it a bit slower than Append().
SourceWriter& Write(const absl::string_view& str);
// Writes a source code snippet read from a file.
//
// All lines of the file at the provided path will be read and written back
// to the output of this writer in regard of its current attributes (e.g.
// the indentation, prefix, etc.)
SourceWriter& WriteFromFile(const std::string& fname,
Env* env = Env::Default());
// Appends a piece of source code.
//
// It is expected that no newline character is present in the data provided,
// otherwise Write() must be used.
SourceWriter& Append(const absl::string_view& str);
// Appends a type to the current line.
//
// The type is written in its simple form (i.e. not prefixed by its package)
// and followed by any parameter types it has enclosed in brackets (<>).
SourceWriter& AppendType(const Type& type);
// Appends a newline character.
//
// Data written after calling this method will start on a new line, in respect
// of the current indentation.
SourceWriter& EndLine();
// Begins a block of source code.
//
// This method appends a new opening brace to the current data and indent the
// next lines according to Google Java Style Guide. The block can optionally
// be preceded by an expression (e.g. Append("if(true)").BeginBlock();)
SourceWriter& BeginBlock(const std::string& expression = "");
// Ends the current block of source code.
//
// This method appends a new closing brace to the current data and outdent the
// next lines back to the margin used before BeginBlock() was invoked.
SourceWriter& EndBlock();
// Begins to write a method.
//
// This method outputs the signature of the Java method from the data passed
// in the 'method' parameter and starts a new block. Modifiers are also passed
// in parameter to define the access scope of this method and, optionally,
// a Javadoc.
SourceWriter& BeginMethod(const Method& method, int modifiers,
const Javadoc* javadoc = nullptr);
// Ends the current method.
//
// This method ends the block of code that has begun when invoking
// BeginMethod() prior to this.
SourceWriter& EndMethod();
// Begins to write the main type of a source file.
//
// This method outputs the declaration of the Java type from the data passed
// in the 'type' parameter and starts a new block. Modifiers are also passed
// in parameter to define the access scope of this type and, optionally,
// a Javadoc.
//
// If not null, all types found in the 'extra_dependencies' list will be
// imported before declaring the new type.
SourceWriter& BeginType(const Type& type, int modifiers,
const std::list<Type>* extra_dependencies = nullptr,
const Javadoc* javadoc = nullptr);
// Begins to write a new inner type.
//
// This method outputs the declaration of the Java type from the data passed
// in the 'type' parameter and starts a new block. Modifiers are also passed
// in parameter to define the accesses and the scope of this type and,
// optionally, a Javadoc.
SourceWriter& BeginInnerType(const Type& type, int modifiers,
const Javadoc* javadoc = nullptr);
// Ends the current type.
//
// This method ends the block of code that has begun when invoking
// BeginType() or BeginInnerType() prior to this.
SourceWriter& EndType();
// Writes a variable as fields of a type.
//
// This method must be called within the definition of a type (see BeginType()
// or BeginInnerType()). Modifiers are also be passed in parameter to define
// the accesses and the scope of this field and, optionally, a Javadoc.
SourceWriter& WriteField(const Variable& field, int modifiers,
const Javadoc* javadoc = nullptr);
protected:
virtual void DoAppend(const absl::string_view& str) = 0;
private:
// A utility base class for visiting elements of a type.
class TypeVisitor {
public:
virtual ~TypeVisitor() = default;
void Visit(const Type& type);
protected:
virtual void DoVisit(const Type& type) = 0;
};
// A utility class for keeping track of declared generics in a given scope.
class GenericNamespace : public TypeVisitor {
public:
GenericNamespace() = default;
explicit GenericNamespace(const GenericNamespace* parent)
: generic_names_(parent->generic_names_) {}
std::list<const Type*> declared_types() {
return declared_types_;
}
protected:
virtual void DoVisit(const Type& type);
private:
std::list<const Type*> declared_types_;
std::set<std::string> generic_names_;
};
// A utility class for collecting a list of import statements to declare.
class TypeImporter : public TypeVisitor {
public:
explicit TypeImporter(const std::string& current_package)
: current_package_(current_package) {}
virtual ~TypeImporter() = default;
const std::set<std::string> imports() { return imports_; }
protected:
virtual void DoVisit(const Type& type);
private:
std::string current_package_;
std::set<std::string> imports_;
};
std::string left_margin_;
std::string line_prefix_;
bool newline_ = true;
std::stack<GenericNamespace*> generic_namespaces_;
SourceWriter& WriteModifiers(int modifiers);
SourceWriter& WriteJavadoc(const Javadoc& javadoc);
SourceWriter& WriteAnnotations(const std::list<Annotation>& annotations);
SourceWriter& WriteGenerics(const std::list<const Type*>& generics);
GenericNamespace* PushGenericNamespace(int modifiers);
void PopGenericNamespace();
};
// A writer that outputs source code into a file.
//
// Note: the writer does not acquire the ownership of the file being passed in
// parameter.
class SourceFileWriter : public SourceWriter {
public:
explicit SourceFileWriter(WritableFile* file) : file_(file) {}
virtual ~SourceFileWriter() = default;
protected:
void DoAppend(const absl::string_view& str) override {
TF_CHECK_OK(file_->Append(str));
}
private:
WritableFile* file_;
};
// A writer that outputs source code into a string buffer.
class SourceBufferWriter : public SourceWriter {
public:
SourceBufferWriter() : owns_buffer_(true), buffer_(new std::string()) {}
explicit SourceBufferWriter(std::string* buffer)
: owns_buffer_(false), buffer_(buffer) {}
virtual ~SourceBufferWriter() {
if (owns_buffer_) delete buffer_;
}
const std::string& str() { return *buffer_; }
protected:
void DoAppend(const absl::string_view& str) override {
buffer_->append(str.begin(), str.end());
}
private:
bool owns_buffer_;
std::string* buffer_;
};
} // namespace java
} // namespace tensorflow
#endif // TENSORFLOW_JAVA_SRC_GEN_CC_SOURCE_WRITER_H_
tensorflow/java/src/gen/cc/source_writer_test.cc
+603
View File
@@ -0,0 +1,603 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/java/src/gen/cc/source_writer.h"
#include <list>
#include "tensorflow/core/platform/path.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/java/src/gen/cc/java_defs.h"
namespace tensorflow {
namespace java {
namespace {
TEST(AppendTest, SingleLineText) {
SourceBufferWriter writer;
writer.Append("You say goodbye and I say hello!");
const char* expected = "You say goodbye and I say hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(AppendTest, MultiLineText) {
SourceBufferWriter writer;
writer.Append("You say goodbye\nand I say hello!");
const char* expected = "You say goodbye\nand I say hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(AppendTest, MultiLineTextWithIndent) {
SourceBufferWriter writer;
writer.Indent(2).Append("You say goodbye\nand I say hello!");
const char* expected = " You say goodbye\nand I say hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(AppendTest, MultiLineTextWithPrefix) {
SourceBufferWriter writer;
writer.Prefix("--").Append("You say goodbye\nand I say hello!");
const char* expected = "--You say goodbye\nand I say hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(AppendTest, MultiLineTextWithIndentAndPrefix) {
SourceBufferWriter writer;
writer.Indent(2).Prefix("--").Append("You say goodbye\nand I say hello!");
const char* expected = " --You say goodbye\nand I say hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteTest, SingleLineText) {
SourceBufferWriter writer;
writer.Write("You say goodbye and I say hello!");
const char* expected = "You say goodbye and I say hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteTest, MultiLineText) {
SourceBufferWriter writer;
writer.Write("You say goodbye\nand I say hello!");
const char* expected = "You say goodbye\nand I say hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteTest, MultiLineTextWithIndent) {
SourceBufferWriter writer;
writer.Indent(2).Write("You say goodbye\nand I say hello!");
const char* expected = " You say goodbye\n and I say hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteTest, MultiLineTextWithPrefix) {
SourceBufferWriter writer;
writer.Prefix("--").Write("You say goodbye\nand I say hello!");
const char* expected = "--You say goodbye\n--and I say hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteTest, MultiLineTextWithIndentAndPrefix) {
SourceBufferWriter writer;
writer.Indent(2).Prefix("--").Write("You say goodbye\nand I say hello!");
const char* expected = " --You say goodbye\n --and I say hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(MarginTest, Basic) {
SourceBufferWriter writer;
writer.Append("You say goodbye").EndLine().Append("and I say hello!");
const char* expected = "You say goodbye\nand I say hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(MarginTest, Indent) {
SourceBufferWriter writer;
writer.Append("You say goodbye")
.EndLine()
.Indent(2)
.Append("and I say hello!");
const char* expected = "You say goodbye\n and I say hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(MarginTest, IndentAndOutdent) {
SourceBufferWriter writer;
writer.Append("You say goodbye")
.EndLine()
.Indent(2)
.Append("and I say hello!")
.EndLine()
.Indent(-2)
.Append("Hello, hello!");
const char* expected = "You say goodbye\n and I say hello!\nHello, hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(MarginTest, Prefix) {
SourceBufferWriter writer;
writer.Append("You say goodbye")
.EndLine()
.Prefix("--")
.Append("and I say hello!");
const char* expected = "You say goodbye\n--and I say hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(MarginTest, PrefixAndRemovePrefix) {
SourceBufferWriter writer;
writer.Append("You say goodbye")
.EndLine()
.Prefix("--")
.Append("and I say hello!")
.EndLine()
.Prefix("")
.Append("Hello, hello!");
const char* expected = "You say goodbye\n--and I say hello!\nHello, hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(MarginTest, IndentAndPrefixAndOutdentAndRemovePrefix) {
SourceBufferWriter writer;
writer.Append("You say goodbye")
.EndLine()
.Indent(2)
.Prefix("--")
.Append("and I say hello!")
.EndLine()
.Indent(-2)
.Prefix("")
.Append("Hello, hello!");
const char* expected = "You say goodbye\n --and I say hello!\nHello, hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(MarginTest, NegativeIndent) {
SourceBufferWriter writer;
writer.Append("You say goodbye")
.EndLine()
.Indent(-10)
.Append("and I say hello!");
const char* expected = "You say goodbye\nand I say hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(MarginTest, CumulativeIndent) {
SourceBufferWriter writer;
writer.Append("You say goodbye")
.EndLine()
.Indent(2)
.Append("and I say hello!")
.EndLine()
.Indent(2)
.Append("Hello, hello!");
const char* expected =
"You say goodbye\n and I say hello!\n Hello, hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(MarginTest, EmptyPrefix) {
SourceBufferWriter writer;
writer.Append("You say goodbye")
.EndLine()
.Prefix("")
.Append("and I say hello!");
const char* expected = "You say goodbye\nand I say hello!";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(StreamTest, BlocksAndLines) {
SourceBufferWriter writer;
writer.Append("int i = 0;").EndLine()
.Append("int j = 10;").EndLine()
.Append("if (true)")
.BeginBlock()
.Append("int aLongWayToTen = 0;").EndLine()
.Append("while (++i <= j)")
.BeginBlock()
.Append("++aLongWayToTen;").EndLine()
.EndBlock()
.EndBlock();
const char* expected =
"int i = 0;\n"
"int j = 10;\n"
"if (true) {\n"
" int aLongWayToTen = 0;\n"
" while (++i <= j) {\n"
" ++aLongWayToTen;\n"
" }\n"
"}\n";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(StreamTest, Types) {
SourceBufferWriter writer;
Type generic = Type::Generic("T").add_supertype(Type::Class("Number"));
writer.AppendType(Type::Int())
.Append(", ")
.AppendType(Type::Class("String"))
.Append(", ")
.AppendType(generic)
.Append(", ")
.AppendType(Type::ListOf(generic))
.Append(", ")
.AppendType(Type::ListOf(Type::IterableOf(generic)))
.Append(", ")
.AppendType(Type::ListOf(Type::Wildcard()));
const char* expected =
"int, String, T, List<T>, List<Iterable<T>>, List<?>";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(StreamTest, FileSnippet) {
SourceBufferWriter writer;
const std::string fname =
tensorflow::io::JoinPath(tensorflow::testing::TensorFlowSrcRoot(),
"java/src/gen/resources/test.java.snippet");
writer.WriteFromFile(fname)
.BeginBlock()
.WriteFromFile(fname)
.EndBlock();
const char* expected =
"// Here is a little snippet\n"
"System.out.println(\"Hello!\");\n"
"{\n"
" // Here is a little snippet\n"
" System.out.println(\"Hello!\");\n"
"}\n";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteType, SimpleClass) {
SourceBufferWriter writer;
Type clazz = Type::Class("Test", "org.tensorflow");
writer.BeginType(clazz, PUBLIC).EndType();
const char* expected =
"package org.tensorflow;\n\n"
"public class Test {\n}\n";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteType, SimpleClassWithDependencies) {
SourceBufferWriter writer;
Type clazz = Type::Class("Test", "org.tensorflow");
std::list<Type> deps;
deps.push_back(Type::Class("TypeA", "org.test.sub"));
deps.push_back(Type::Class("TypeA", "org.test.sub")); // a second time
deps.push_back(Type::Class("TypeB", "org.other"));
deps.push_back(Type::Class("SamePackageType", "org.tensorflow"));
deps.push_back(Type::Class("NoPackageType"));
writer.BeginType(clazz, PUBLIC, &deps).EndType();
const char* expected =
"package org.tensorflow;\n\n"
"import org.other.TypeB;\n"
"import org.test.sub.TypeA;\n\n"
"public class Test {\n}\n";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteType, AnnotatedAndDocumentedClass) {
SourceBufferWriter writer;
Type clazz = Type::Class("Test", "org.tensorflow");
Javadoc clazz_doc = Javadoc::Create("Javadoc test")
.details("This is a\nmultiline description.");
clazz.add_annotation(Annotation::Create("Bean"));
clazz.add_annotation(Annotation::Create("SuppressWarnings")
.attributes("\"rawtypes\""));
writer.BeginType(clazz, PUBLIC, nullptr, &clazz_doc).EndType();
const char* expected =
"package org.tensorflow;\n\n"
"/**\n"
" * Javadoc test\n"
" * <p>\n"
" * This is a\n"
" * multiline description.\n"
" */\n"
"@Bean\n"
"@SuppressWarnings(\"rawtypes\")\n"
"public class Test {\n}\n";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteType, ParameterizedClass) {
SourceBufferWriter writer;
Type clazz = Type::Class("Test", "org.tensorflow");
clazz.add_parameter(Type::Generic("T"));
clazz.add_parameter(Type::Generic("U").add_supertype(Type::Class("Number")));
writer.BeginType(clazz, PUBLIC).EndType();
const char* expected =
"package org.tensorflow;\n\n"
"public class Test<T, U extends Number> {\n}\n";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteType, ParameterizedClassAndSupertypes) {
SourceBufferWriter writer;
Type clazz = Type::Class("Test", "org.tensorflow");
Type type_t = Type::Generic("T");
clazz.add_parameter(type_t);
Type type_u = Type::Generic("U").add_supertype(Type::Class("Number"));
clazz.add_parameter(type_u);
clazz.add_supertype(Type::Interface("Parameterizable").add_parameter(type_u));
clazz.add_supertype(Type::Interface("Runnable"));
clazz.add_supertype(Type::Class("SuperTest").add_parameter(type_t));
writer.BeginType(clazz, PUBLIC).EndType();
const char* expected =
"package org.tensorflow;\n\n"
"public class Test<T, U extends Number>"
" extends SuperTest<T> implements Parameterizable<U>, Runnable {\n}\n";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteType, ParameterizedClassFields) {
SourceBufferWriter writer;
Type clazz = Type::Class("Test", "org.tensorflow");
Type type_t = Type::Generic("T").add_supertype(Type::Class("Number"));
clazz.add_parameter(type_t);
Variable field1 = Variable::Create("field1", Type::Class("String"));
Variable field2 = Variable::Create("field2", Type::Class("String"));
Variable field3 = Variable::Create("field3", type_t);
Javadoc field3_doc = Javadoc::Create("This variable is documented");
writer.BeginType(clazz, PUBLIC)
.WriteField(field1, STATIC | PUBLIC | FINAL)
.WriteField(field2, PRIVATE)
.WriteField(field3, PRIVATE, &field3_doc)
.EndType();
const char* expected =
"package org.tensorflow;\n\n"
"public class Test<T extends Number> {\n"
" public static final String field1;\n"
" private String field2;\n"
" /** This variable is documented */\n"
" private T field3;\n"
"}\n";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteType, SimpleInnerClass) {
SourceBufferWriter writer;
Type clazz = Type::Class("Test", "org.tensorflow");
Type inner_class = Type::Class("InnerTest");
writer.BeginType(clazz, PUBLIC)
.BeginInnerType(inner_class, PUBLIC)
.EndType()
.EndType();
const char* expected =
"package org.tensorflow;\n\n"
"public class Test {\n"
" \n"
" public class InnerTest {\n"
" }\n"
"}\n";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteType, StaticParameterizedInnerClass) {
SourceBufferWriter writer;
Type clazz = Type::Class("Test", "org.tensorflow");
Type type_t = Type::Generic("T").add_supertype(Type::Class("Number"));
clazz.add_parameter(type_t);
Type inner_class = Type::Class("InnerTest");
inner_class.add_parameter(type_t);
writer.BeginType(clazz, PUBLIC)
.BeginInnerType(inner_class, PUBLIC | STATIC)
.EndType()
.EndType();
const char* expected =
"package org.tensorflow;\n\n"
"public class Test<T extends Number> {\n"
" \n"
" public static class InnerTest<T extends Number> {\n"
" }\n"
"}\n";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteMethod, SimpleMethod) {
SourceBufferWriter writer;
Type clazz = Type::Class("Test", "org.tensorflow");
Method method = Method::Create("doNothing", Type::Void());
writer.BeginType(clazz, PUBLIC)
.BeginMethod(method, PUBLIC)
.EndMethod()
.EndType();
const char* expected =
"package org.tensorflow;\n\n"
"public class Test {\n"
" \n"
" public void doNothing() {\n"
" }\n"
"}\n";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteMethod, AnnotatedAndDocumentedMethod) {
SourceBufferWriter writer;
Type clazz = Type::Class("Test", "org.tensorflow");
Method method = Method::Create("doNothing", Type::Void());
Javadoc method_doc =
Javadoc::Create("Javadoc test")
.details("This method has a\nmultiline description.");
method.add_annotation(Annotation::Create("Override"));
method.add_annotation(Annotation::Create("SuppressWarnings")
.attributes("\"rawtypes\""));
writer.BeginType(clazz, PUBLIC)
.BeginMethod(method, PUBLIC, &method_doc)
.EndMethod()
.EndType();
const char* expected =
"package org.tensorflow;\n\n"
"public class Test {\n"
" \n"
" /**\n"
" * Javadoc test\n"
" * <p>\n"
" * This method has a\n"
" * multiline description.\n"
" */\n"
" @Override\n"
" @SuppressWarnings(\"rawtypes\")\n"
" public void doNothing() {\n"
" }\n"
"}\n";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteMethod, DocumentedMethodWithArguments) {
SourceBufferWriter writer;
Type clazz = Type::Class("Test", "org.tensorflow");
Variable reverse = Variable::Create("reverse", Type::Boolean());
Method method = Method::Create("boolToInt", Type::Int());
method.add_argument(Variable::Create("b", Type::Boolean()));
method.add_argument(reverse);
Javadoc method_doc =
Javadoc::Create("Converts a boolean to an int")
.details("This method will convert\na boolean to an int")
.add_param_tag(reverse.name(), "if true, value is reversed")
.add_tag("return", "int value for this boolean");
writer.BeginType(clazz, PUBLIC)
.BeginMethod(method, PUBLIC, &method_doc)
.Append("if (b && !reverse)")
.BeginBlock()
.Append("return 1;")
.EndLine()
.EndBlock()
.Append("return 0;")
.EndLine()
.EndMethod()
.EndType();
const char* expected =
"package org.tensorflow;\n\n"
"public class Test {\n"
" \n"
" /**\n"
" * Converts a boolean to an int\n"
" * <p>\n"
" * This method will convert\n"
" * a boolean to an int\n"
" * \n"
" * @param reverse if true, value is reversed\n"
" * @return int value for this boolean\n"
" */\n"
" public int boolToInt(boolean b, boolean reverse) {\n"
" if (b && !reverse) {\n"
" return 1;\n"
" }\n"
" return 0;\n"
" }\n"
"}\n";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteMethod, ParameterizedMethod) {
SourceBufferWriter writer;
Type clazz = Type::Class("Test", "org.tensorflow");
Type type_t = Type::Generic("T").add_supertype(Type::Class("Number"));
clazz.add_parameter(type_t);
Method method = Method::Create("doNothing", type_t);
writer.BeginType(clazz, PUBLIC)
.BeginMethod(method, PUBLIC)
.Append("return null;")
.EndLine()
.EndMethod()
.EndType();
const char* expected =
"package org.tensorflow;\n\n"
"public class Test<T extends Number> {\n"
" \n"
" public T doNothing() {\n"
" return null;\n"
" }\n"
"}\n";
ASSERT_STREQ(expected, writer.str().data());
}
TEST(WriteMethod, StaticParameterizedMethod) {
SourceBufferWriter writer;
Type clazz = Type::Class("Test", "org.tensorflow");
Type type_t = Type::Generic("T").add_supertype(Type::Class("Number"));
clazz.add_parameter(type_t);
Method method = Method::Create("doNothing", type_t);
writer.BeginType(clazz, PUBLIC)
.BeginMethod(method, PUBLIC | STATIC)
.Append("return null;")
.EndLine()
.EndMethod()
.EndType();
const char* expected =
"package org.tensorflow;\n\n"
"public class Test<T extends Number> {\n"
" \n"
" public static <T extends Number> T doNothing() {\n"
" return null;\n"
" }\n"
"}\n";
ASSERT_STREQ(expected, writer.str().data());
}
} // namespace
} // namespace java
} // namespace tensorflow
tensorflow/java/src/gen/gen_ops.bzl
+65
View File
@@ -0,0 +1,65 @@
load(
"//tensorflow:tensorflow.bzl",
"tf_binary_additional_srcs",
)
# Generate Java wrapper classes for all registered core operations and package
# them into a single source archive (.srcjar).
#
# For example:
# tf_java_op_gen_srcjar("gen_sources", ":gen_tool", "my.package")
#
# will create a genrule named "gen_sources" that generates source files under
# ops/src/main/java/my/package/**/*.java
#
# and then archive those source files into
# ops/gen_sources.srcjar
#
def tf_java_op_gen_srcjar(
name,
gen_tool,
base_package,
api_def_srcs = [],
out_dir = "ops/",
out_src_dir = "src/main/java/",
visibility = ["//tensorflow/java:__pkg__"]):
gen_cmds = ["rm -rf $(@D)"] # Always start from fresh when generating source files
srcs = api_def_srcs[:]
if not api_def_srcs:
api_def_args_str = ","
else:
api_def_args = []
for api_def_src in api_def_srcs:
# Add directory of the first ApiDef source to args.
# We are assuming all ApiDefs in a single api_def_src are in the
# same directory.
api_def_args.append(
"$$(dirname $$(echo $(locations " + api_def_src +
") | cut -d\" \" -f1))",
)
api_def_args_str = ",".join(api_def_args)
gen_cmds += ["$(location " + gen_tool + ")" +
" --output_dir=$(@D)/" + out_src_dir +
" --base_package=" + base_package +
" --api_dirs=" + api_def_args_str]
# Generate a source archive containing generated code for these ops.
gen_srcjar = out_dir + name + ".srcjar"
gen_cmds += ["$(JAVABASE)/bin/jar cMf $(location :" + gen_srcjar + ") -C $(@D) src"]
native.genrule(
name = name,
srcs = srcs,
outs = [gen_srcjar],
tools = [
# copybara:uncomment_begin(using system-provided in OSS build)
# "//third_party/java/jar:jar",
# "//third_party/java/jdk:jdk",
# copybara:uncomment_end
gen_tool,
] + tf_binary_additional_srcs(),
toolchains = ["@bazel_tools//tools/jdk:current_host_java_runtime"],
cmd = " && ".join(gen_cmds),
)
tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java
+490
View File
@@ -0,0 +1,490 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.processor;
import com.google.common.base.CaseFormat;
import com.google.common.base.Strings;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.squareup.javapoet.ClassName;
import com.squareup.javapoet.FieldSpec;
import com.squareup.javapoet.JavaFile;
import com.squareup.javapoet.MethodSpec;
import com.squareup.javapoet.ParameterSpec;
import com.squareup.javapoet.ParameterizedTypeName;
import com.squareup.javapoet.TypeName;
import com.squareup.javapoet.TypeSpec;
import com.squareup.javapoet.TypeVariableName;
import com.squareup.javapoet.WildcardTypeName;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.processing.AbstractProcessor;
import javax.annotation.processing.Filer;
import javax.annotation.processing.Messager;
import javax.annotation.processing.ProcessingEnvironment;
import javax.annotation.processing.RoundEnvironment;
import javax.lang.model.SourceVersion;
import javax.lang.model.element.AnnotationMirror;
import javax.lang.model.element.AnnotationValue;
import javax.lang.model.element.Element;
import javax.lang.model.element.ExecutableElement;
import javax.lang.model.element.Modifier;
import javax.lang.model.element.TypeElement;
import javax.lang.model.element.TypeParameterElement;
import javax.lang.model.element.VariableElement;
import javax.lang.model.type.TypeMirror;
import javax.lang.model.type.TypeVariable;
import javax.lang.model.util.ElementFilter;
import javax.lang.model.util.Elements;
import javax.tools.Diagnostic.Kind;
/**
* A compile-time Processor that aggregates classes annotated with {@link
* org.tensorflow.op.annotation.Operator} and generates the {@code Ops} convenience API. Please
* refer to the {@link org.tensorflow.op.annotation.Operator} annotation for details about the API
* generated for each annotated class.
*
* <p>Note that this processor can only be invoked once, in a single compilation run that includes
* all the {@code Operator} annotated source classes. The reason is that the {@code Ops} API is an
* "aggregating" API, and annotation processing does not permit modifying an already generated
* class.
*
* @see org.tensorflow.op.annotation.Operator
*/
public final class OperatorProcessor extends AbstractProcessor {
@Override
public SourceVersion getSupportedSourceVersion() {
return SourceVersion.latest();
}
@Override
public synchronized void init(ProcessingEnvironment processingEnv) {
super.init(processingEnv);
messager = processingEnv.getMessager();
filer = processingEnv.getFiler();
elements = processingEnv.getElementUtils();
}
@Override
public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) {
// Nothing needs to be done at the end of all rounds.
if (roundEnv.processingOver()) {
return false;
}
// Nothing to look at in this round.
if (annotations.size() == 0) {
return false;
}
// We expect to be registered for exactly one annotation.
if (annotations.size() != 1) {
throw new IllegalStateException(
"Unexpected - multiple annotations registered: " + annotations);
}
TypeElement annotation = annotations.iterator().next();
Set<? extends Element> annotated = roundEnv.getElementsAnnotatedWith(annotation);
// If there are no annotated elements, claim the annotation but do nothing.
if (annotated.size() == 0) {
return false;
}
// This processor has to aggregate all op classes in one round, as it generates a single Ops
// API class which cannot be modified once generated. If we find an annotation after we've
// generated our code, flag the location of each such class.
if (hasRun) {
for (Element e : annotated) {
error(
e,
"The Operator processor has already processed @Operator annotated sources\n"
+ "and written out an Ops API. It cannot process additional @Operator sources.\n"
+ "One reason this can happen is if other annotation processors generate\n"
+ "new @Operator source files.");
}
return false;
}
// Collect all classes tagged with our annotation.
Multimap<String, MethodSpec> groupedMethods = HashMultimap.create();
if (!collectOpsMethods(roundEnv, groupedMethods, annotation)) {
return false;
}
// Nothing to do when there are no tagged classes.
if (groupedMethods.isEmpty()) {
return false;
}
// Validate operator classes and generate Op API.
writeApi(groupedMethods);
hasRun = true;
return false;
}
@Override
public Set<String> getSupportedAnnotationTypes() {
return Collections.singleton("org.tensorflow.op.annotation.Operator");
}
private static final Pattern JAVADOC_TAG_PATTERN =
Pattern.compile("@(?:param|return|throws|exception|see)\\s+.*");
private static final TypeName T_OP = ClassName.get("org.tensorflow.op", "Op");
private static final TypeName T_OPS = ClassName.get("org.tensorflow.op", "Ops");
private static final TypeName T_OPERATOR =
ClassName.get("org.tensorflow.op.annotation", "Operator");
private static final TypeName T_SCOPE = ClassName.get("org.tensorflow.op", "Scope");
private static final TypeName T_EXEC_ENV =
ClassName.get("org.tensorflow", "ExecutionEnvironment");
private static final TypeName T_EAGER_SESSION = ClassName.get("org.tensorflow", "EagerSession");
private static final TypeName T_STRING = ClassName.get(String.class);
// Operand<?>
private static final TypeName T_OPERAND =
ParameterizedTypeName.get(
ClassName.get("org.tensorflow", "Operand"), WildcardTypeName.subtypeOf(Object.class));
// Iterable<Operand<?>>
private static final TypeName T_ITERABLE_OPERAND =
ParameterizedTypeName.get(ClassName.get(Iterable.class), T_OPERAND);
private Filer filer;
private Messager messager;
private Elements elements;
private boolean hasRun = false;
private void error(Element e, String message, Object... args) {
if (args != null && args.length > 0) {
message = String.format(message, args);
}
messager.printMessage(Kind.ERROR, message, e);
}
private void write(TypeSpec spec) {
try {
JavaFile.builder("org.tensorflow.op", spec).skipJavaLangImports(true).build().writeTo(filer);
} catch (IOException e) {
throw new AssertionError(e);
}
}
private void writeApi(Multimap<String, MethodSpec> groupedMethods) {
Map<String, ClassName> groups = new HashMap<>();
// Generate a API class for each group collected other than the default one (= empty string)
for (Map.Entry<String, Collection<MethodSpec>> entry : groupedMethods.asMap().entrySet()) {
if (!entry.getKey().isEmpty()) {
TypeSpec groupClass = buildGroupClass(entry.getKey(), entry.getValue());
write(groupClass);
groups.put(entry.getKey(), ClassName.get("org.tensorflow.op", groupClass.name));
}
}
// Generate the top API class, adding any methods added to the default group
TypeSpec topClass = buildTopClass(groups, groupedMethods.get(""));
write(topClass);
}
private boolean collectOpsMethods(
RoundEnvironment roundEnv,
Multimap<String, MethodSpec> groupedMethods,
TypeElement annotation) {
boolean result = true;
for (Element e : roundEnv.getElementsAnnotatedWith(annotation)) {
// @Operator can only apply to types, so e must be a TypeElement.
if (!(e instanceof TypeElement)) {
error(
e,
"@Operator can only be applied to classes, but this is a %s",
e.getKind().toString());
result = false;
continue;
}
TypeElement opClass = (TypeElement) e;
// Skip deprecated operations for now, as we do not guarantee API stability yet
if (opClass.getAnnotation(Deprecated.class) == null) {
collectOpMethods(groupedMethods, opClass, annotation);
}
}
return result;
}
private void collectOpMethods(
Multimap<String, MethodSpec> groupedMethods, TypeElement opClass, TypeElement annotation) {
AnnotationMirror am = getAnnotationMirror(opClass, annotation);
String groupName = getAnnotationElementValueAsString("group", am);
String methodName = getAnnotationElementValueAsString("name", am);
ClassName opClassName = ClassName.get(opClass);
if (Strings.isNullOrEmpty(methodName)) {
methodName = CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_CAMEL, opClassName.simpleName());
}
// Build a method for each @Operator found in the class path. There should be one method per
// operation factory called
// "create", which takes in parameter a scope and, optionally, a list of arguments
for (ExecutableElement opMethod : ElementFilter.methodsIn(opClass.getEnclosedElements())) {
if (opMethod.getModifiers().contains(Modifier.STATIC)
&& opMethod.getSimpleName().contentEquals("create")) {
MethodSpec method = buildOpMethod(methodName, opClassName, opMethod);
groupedMethods.put(groupName, method);
}
}
}
private MethodSpec buildOpMethod(
String methodName, ClassName opClassName, ExecutableElement factoryMethod) {
MethodSpec.Builder builder =
MethodSpec.methodBuilder(methodName)
.addModifiers(Modifier.PUBLIC)
.returns(TypeName.get(factoryMethod.getReturnType()))
.varargs(factoryMethod.isVarArgs())
.addJavadoc("$L", buildOpMethodJavadoc(opClassName, factoryMethod));
for (TypeParameterElement tp : factoryMethod.getTypeParameters()) {
TypeVariableName tvn = TypeVariableName.get((TypeVariable) tp.asType());
builder.addTypeVariable(tvn);
}
for (TypeMirror thrownType : factoryMethod.getThrownTypes()) {
builder.addException(TypeName.get(thrownType));
}
StringBuilder call = new StringBuilder("return $T.create(scope");
boolean first = true;
for (VariableElement param : factoryMethod.getParameters()) {
ParameterSpec p = ParameterSpec.get(param);
if (first) {
first = false;
continue;
}
call.append(", ");
call.append(p.name);
builder.addParameter(p);
}
call.append(")");
builder.addStatement(call.toString(), opClassName);
return builder.build();
}
private String buildOpMethodJavadoc(ClassName opClassName, ExecutableElement factoryMethod) {
StringBuilder javadoc = new StringBuilder();
javadoc.append("Builds an {@link ").append(opClassName.simpleName()).append("} operation\n\n");
// Add all javadoc tags found in the operator factory method but the first one, which should be
// in all cases the
// 'scope' parameter that is implicitly passed by this API
Matcher tagMatcher = JAVADOC_TAG_PATTERN.matcher(elements.getDocComment(factoryMethod));
boolean firstParam = true;
while (tagMatcher.find()) {
String tag = tagMatcher.group();
if (tag.startsWith("@param") && firstParam) {
firstParam = false;
} else {
javadoc.append(tag).append('\n');
}
}
javadoc.append("@see ").append(opClassName).append("\n");
return javadoc.toString();
}
private static TypeSpec buildGroupClass(String group, Collection<MethodSpec> methods) {
MethodSpec.Builder ctorBuilder =
MethodSpec.constructorBuilder()
.addParameter(T_SCOPE, "scope")
.addStatement("this.scope = scope");
TypeSpec.Builder builder =
TypeSpec.classBuilder(CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_CAMEL, group) + "Ops")
.addModifiers(Modifier.PUBLIC, Modifier.FINAL)
.addJavadoc(
"An API for building {@code $L} operations as {@link $T Op}s\n\n"
+ "@see {@link $T}\n",
group,
T_OP,
T_OPS)
.addMethods(methods)
.addMethod(ctorBuilder.build());
builder.addField(
FieldSpec.builder(T_SCOPE, "scope").addModifiers(Modifier.PRIVATE, Modifier.FINAL).build());
return builder.build();
}
private static TypeSpec buildTopClass(
Map<String, ClassName> groupToClass, Collection<MethodSpec> methods) {
MethodSpec.Builder ctorBuilder =
MethodSpec.constructorBuilder()
.addModifiers(Modifier.PRIVATE)
.addParameter(T_SCOPE, "scope")
.addStatement("this.scope = scope", T_SCOPE);
for (Map.Entry<String, ClassName> entry : groupToClass.entrySet()) {
ctorBuilder.addStatement("$L = new $T(scope)", entry.getKey(), entry.getValue());
}
TypeSpec.Builder opsBuilder =
TypeSpec.classBuilder("Ops")
.addModifiers(Modifier.PUBLIC, Modifier.FINAL)
.addJavadoc(
"An API for building operations as {@link $T Op}s\n<p>\n"
+ "Any operation wrapper found in the classpath properly annotated as an"
+ "{@link $T @Operator} is exposed\n"
+ "by this API or one of its subgroup.\n<p>Example usage:\n<pre>{@code\n"
+ "try (Graph g = new Graph()) {\n"
+ " Ops ops = Ops.create(g);\n"
+ " // Operations are typed classes with convenience\n"
+ " // builders in Ops.\n"
+ " Constant three = ops.constant(3);\n"
+ " // Single-result operations implement the Operand\n"
+ " // interface, so this works too.\n"
+ " Operand four = ops.constant(4);\n"
+ " // Most builders are found within a group, and accept\n"
+ " // Operand types as operands\n"
+ " Operand nine = ops.math.add(four, ops.constant(5));\n"
+ " // Multi-result operations however offer methods to\n"
+ " // select a particular result for use.\n"
+ " Operand result = \n"
+ " ops.math.add(ops.unique(s, a).y(), b);\n"
+ " // Optional attributes\n"
+ " ops.linalg.matMul(a, b, MatMul.transposeA(true));\n"
+ " // Naming operators\n"
+ " ops.withName(\"foo\").constant(5); // name \"foo\"\n"
+ " // Names can exist in a hierarchy\n"
+ " Ops sub = ops.withSubScope(\"sub\");\n"
+ " sub.withName(\"bar\").constant(4); // \"sub/bar\"\n"
+ "}\n"
+ "}</pre>\n",
T_OP,
T_OPERATOR)
.addMethods(methods)
.addMethod(ctorBuilder.build());
opsBuilder.addMethod(
MethodSpec.methodBuilder("withSubScope")
.addModifiers(Modifier.PUBLIC)
.addParameter(T_STRING, "childScopeName")
.returns(T_OPS)
.addStatement("return new $T(scope.withSubScope(childScopeName))", T_OPS)
.addJavadoc(
"Returns an API that builds operations with the provided name prefix.\n"
+ "\n@see {@link $T#withSubScope(String)}\n",
T_SCOPE)
.build());
opsBuilder.addMethod(
MethodSpec.methodBuilder("withName")
.addModifiers(Modifier.PUBLIC)
.addParameter(T_STRING, "opName")
.returns(T_OPS)
.addStatement("return new Ops(scope.withName(opName))")
.addJavadoc(
"Returns an API that uses the provided name for an op.\n\n"
+ "@see {@link $T#withName(String)}\n",
T_SCOPE)
.build());
opsBuilder.addMethod(
MethodSpec.methodBuilder("withControlDependencies")
.addModifiers(Modifier.PUBLIC)
.addParameter(T_ITERABLE_OPERAND, "controls")
.returns(T_OPS)
.addStatement("return new Ops(scope.withControlDependencies(controls))")
.addJavadoc(
"Returns an API that adds operations to the graph with the provided control"
+ " dependencies.\n\n"
+ "@see {@link $T#withControlDependencies(Iterable<Operand<?>>)}\n",
T_SCOPE)
.build());
opsBuilder.addField(
FieldSpec.builder(T_SCOPE, "scope").addModifiers(Modifier.PRIVATE, Modifier.FINAL).build());
opsBuilder.addMethod(
MethodSpec.methodBuilder("scope")
.addModifiers(Modifier.PUBLIC, Modifier.FINAL)
.returns(T_SCOPE)
.addStatement("return scope")
.addJavadoc("Returns the current {@link $T scope} of this API\n", T_SCOPE)
.build());
for (Map.Entry<String, ClassName> entry : groupToClass.entrySet()) {
opsBuilder.addField(
FieldSpec.builder(entry.getValue(), entry.getKey())
.addModifiers(Modifier.PUBLIC, Modifier.FINAL)
.build());
opsBuilder.addMethod(
MethodSpec.methodBuilder(entry.getKey())
.addModifiers(Modifier.PUBLIC, Modifier.FINAL)
.returns(entry.getValue())
.addStatement("return $L", entry.getKey())
.addJavadoc("Returns an API for building {@code $L} operations\n", entry.getKey())
.build());
}
opsBuilder.addMethod(
MethodSpec.methodBuilder("create")
.addModifiers(Modifier.PUBLIC, Modifier.STATIC)
.addParameter(T_EXEC_ENV, "env")
.returns(T_OPS)
.addStatement("return new Ops(new $T(env))", T_SCOPE)
.addJavadoc(
"Creates an API for building operations in the provided execution environment\n")
.build());
opsBuilder.addMethod(
MethodSpec.methodBuilder("create")
.addModifiers(Modifier.PUBLIC, Modifier.STATIC)
.returns(T_OPS)
.addStatement("return new Ops(new $T($T.getDefault()))", T_SCOPE, T_EAGER_SESSION)
.addJavadoc(
"Creates an API for building operations in the default eager execution"
+ " environment\n\n"
+ "<p>Invoking this method is equivalent to {@code"
+ " Ops.create(EagerSession.getDefault())}.\n")
.build());
return opsBuilder.build();
}
private static AnnotationMirror getAnnotationMirror(Element element, TypeElement annotation) {
for (AnnotationMirror am : element.getAnnotationMirrors()) {
if (am.getAnnotationType().asElement().equals(annotation)) {
return am;
}
}
throw new IllegalArgumentException(
"Annotation "
+ annotation.getSimpleName()
+ " not present on element "
+ element.getSimpleName());
}
private static String getAnnotationElementValueAsString(String elementName, AnnotationMirror am) {
for (Map.Entry<? extends ExecutableElement, ? extends AnnotationValue> entry :
am.getElementValues().entrySet()) {
if (entry.getKey().getSimpleName().contentEquals(elementName)) {
return entry.getValue().getValue().toString();
}
}
return "";
}
}
tensorflow/java/src/gen/perl/tftypes-runall.pl
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/perl
#
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================
my $script = $0;
my $dir = `dirname $script`;
chomp $dir;
my $gen = "$dir/..";
my $tfjavasrc = "$gen/..";
my $rsrc = "$gen/resources";
my $root = "$tfjavasrc/main/java";
my $pkg = "$root/org/tensorflow";
sub locchk {
(my $f) = @_;
if (! -r $f) {
print STDERR "Script tftypes-runall seems to be located in the wrong place (could not find $f)\n";
exit 1;
}
}
&locchk("$gen");
&locchk("$tfjavasrc/gen");
&locchk("$dir/tftypes.pl");
&locchk("$rsrc/tftypes.csv");
system("perl $dir/tftypes.pl -t $rsrc/tftypes.csv $pkg/types");
system("perl $dir/tftypes.pl -c $rsrc/tftypes.csv $rsrc/Tensors.java.tmpl > $pkg/Tensors.java");
tensorflow/java/src/gen/perl/tftypes.pl
+187
View File
@@ -0,0 +1,187 @@
#!/usr/bin/perl
#
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================
use strict;
my $copyright =
'/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
';
my $count;
my $option = '-t', my $template;
sub usage {
print "Usage: tftypes [-ctdT] <type desc file> <tmpl file>\n\n"
."This script generates parts of various .java files that depend on which"
."TensorFlow types are supported by the Java API and how much. For each"
."such .java file, there is a .tmpl file in the same source directory in"
."which the strings \@TYPEINFO\@ and \@IMPORTS\@ are replaced with"
."appropriate Java code. Output code is sent to standard output.\n\n";
print "Modulo putting in the correct directory names, it can be invoked as follows:\n";
print "tftypes -c tftypes.csv Tensors.java.tmpl > Tensors.java\n";
print "tftypes -t tftypes.csv <dir> [outputs files to dir]\n";
}
if ($ARGV[0] =~ m/^-/) {
$option = shift;
}
my $typedesc = shift;
my $tmpl = shift;
my $dirname;
if ($option eq '-t') {
$dirname = $tmpl;
}
open (TMPL, "<$tmpl") || die "Cannot open $tmpl for reading\n";
my $text = do { local $/; <TMPL> };
my %jtypecount;
my $typeinfo, my $imports;
open (TYPEDESC, $typedesc);
my @info = ([]);
sub trim {
(my $ret) = @_;
$ret =~ s/^\s*//g;
$ret =~ s/\s*$//g;
return $ret;
}
while (<TYPEDESC>) {
chomp;
my $line = $_;
if ($line =~ m/^TF type/) { next }
$line =~ s/\r$//;
my @items = split /,/, $line, 6;
for (my $i = 0; $i <= $#items; $i++) {
$items[$i] = trim $items[$i];
}
my $jtype = $items[2];
$jtypecount{$jtype}++;
if ($jtypecount{$jtype} > 1) {
# currently allowing Java types to stand for more than one TF type, but
# may want to revisit this.
# print STDERR "Ambiguous Java type for $name : $jtype\n";
# exit 1
}
push @info, \@items;
}
sub article {
(my $s) = @_;
if (substr($s, 0, 1) =~ m/^[aeoiu8]$/i) {
return "an $s"
} else {
return "a $s"
}
}
for (my $i = 1; $i <= $#info; $i++) {
(my $name, my $builtin, my $jtype, my $creat, my $default, my $desc) =
@{$info[$i]};
my $tfname = $name;
my $ucname = uc $name;
print STDERR "$name $desc\n";
if ($option eq '-t') {
if ($jtype eq '') { next }
if ($builtin eq 'y') { next }
# Generate class declarations
# print STDERR "Creating $dirname/$tfname.java\n";
open (CLASSFILE, ">$dirname/$tfname.java") || die "Can't open $tfname.java";
print CLASSFILE $copyright, "\n";
# print CLASSFILE "// GENERATED FILE. To update, edit tftypes.pl instead.\n\n";
my $fulldesc = article($desc);
print CLASSFILE "package org.tensorflow.types;\n\n";
print CLASSFILE "/** Represents $fulldesc. */\n"
."public class $tfname {\n"
." private $tfname() {\n"
." }\n"
."}\n";
close(CLASSFILE);
} elsif ($option eq '-c') {
# Generate creator declarations for Tensors.java
if ($jtype ne '' && $creat eq 'y') {
for (my $brackets = '', my $rank = 0; length $brackets <= 12; $brackets .= '[]', $rank++) {
my $datainfo = " * \@param data An array containing the values to put into the new tensor.\n"
." * The dimensions of the new tensor will match those of the array.\n";
if ($rank == 0) {
$datainfo = " * \@param data The value to put into the new scalar tensor.\n"
}
my $trank = $rank;
if ($tfname eq 'String') {
$trank = $rank-1;
next if $trank < 0;
$datainfo = " * \@param data An array containing the data to put into the new tensor.\n"
." * String elements are sequences of bytes from the last array dimension.\n";
}
my $intro = ($trank > 0)
? "Creates a rank-$trank tensor of {\@code $jtype} elements."
: "Creates a scalar tensor containing a single {\@code $jtype} element.";
$typeinfo .=
" /**\n"
." * $intro\n"
." * \n"
.$datainfo
." */\n"
." public static Tensor<$tfname> create($jtype$brackets data) {\n"
." return Tensor.create(data, $tfname.class);\n"
." }\n\n";
}
}
if ($text =~ m/\b$tfname\b/ && $builtin eq 'n' && $creat eq 'y') {
$imports .= "import org.tensorflow.types.$tfname;\n";
}
}
}
if ($option ne '-t') {
# print "// GENERATED FILE. Edits to this file will be lost -- edit $tmpl instead.\n";
$text =~ s/\@TYPEINFO\@/$typeinfo/;
$text =~ s/\@IMPORTS\@/$imports/;
print $text;
}
tensorflow/java/src/gen/resources/META-INF/services/javax.annotation.processing.Processor
+1
View File
@@ -0,0 +1 @@
org.tensorflow.processor.OperatorProcessor
tensorflow/java/src/gen/resources/Tensors.java.tmpl
+31
View File
@@ -0,0 +1,31 @@
package org.tensorflow;
import static java.nio.charset.StandardCharsets.UTF_8;
import org.tensorflow.Tensor;
@IMPORTS@
/**
* Type-safe factory methods for creating {@link Tensor} objects.
*/
public final class Tensors {
private Tensors() {}
/** Creates a scalar String tensor using the default, UTF-8 encoding.
*
* @param data The string to put into the new scalar tensor.
*/
public static Tensor<String> create(String data) {
return Tensor.create(data.getBytes(UTF_8), String.class);
}
/** Creates a scalar String tensor using a specified encoding.
*
* @param charset The encoding from String to bytes.
* @param data The string to put into the new scalar tensor.
*/
public static Tensor<String> create(String data, java.nio.charset.Charset charset) {
return Tensor.create(data.getBytes(charset), String.class);
}
@TYPEINFO@}
tensorflow/java/src/gen/resources/test.java.snippet
+2
View File
@@ -0,0 +1,2 @@
// Here is a little snippet
System.out.println("Hello!");
tensorflow/java/src/gen/resources/tftypes.csv
+21
View File
@@ -0,0 +1,21 @@
TF type,Builtin,Java type,Creator?,Zero value,Description
Float,y,float,y,0f,32-bit single precision floating point number
Double,y,double,y,0.0,64-bit double precision floating point number
Integer,y,int,y,0,32-bit signed integer
UInt8,n,byte,n,(byte)0,8-bit unsigned integer
Short,y,,n,(short)0,16-bit signed integer
Byte,y,,n,(byte)0,8-bit signed integer
String,y,byte,y,,arbitrary sequence of bytes
Complex64,n,,n,,single-precision complex number
Long,y,long,y,0L,64-bit signed integer
Boolean,y,boolean,y,false,boolean
QInt8,n,,n,,quantized int8
QUInt8,n,,n,,quantized uint8
QInt32,n,,n,,quantized int32
BFloat16,n,,n,,float32 truncated to 16 bits. Only for cast ops.
QInt16,n,,n,,quantized int16
QUInt16,n,,n,,quantized uint16
UInt16,n,,n,,16-bit unsigned integer
Complex128,n,,n,,double-precision complex number
Half,n,,n,,
Resource,n,,n,,
1 TF type Builtin Java type Creator? Zero value Description
2 Float y float y 0f 32-bit single precision floating point number
3 Double y double y 0.0 64-bit double precision floating point number
4 Integer y int y 0 32-bit signed integer
5 UInt8 n byte n (byte)0 8-bit unsigned integer
6 Short y n (short)0 16-bit signed integer
7 Byte y n (byte)0 8-bit signed integer
8 String y byte y arbitrary sequence of bytes
9 Complex64 n n single-precision complex number
10 Long y long y 0L 64-bit signed integer
11 Boolean y boolean y false boolean
12 QInt8 n n quantized int8
13 QUInt8 n n quantized uint8
14 QInt32 n n quantized int32
15 BFloat16 n n float32 truncated to 16 bits. Only for cast ops.
16 QInt16 n n quantized int16
17 QUInt16 n n quantized uint16
18 UInt16 n n 16-bit unsigned integer
19 Complex128 n n double-precision complex number
20 Half n n
21 Resource n n
tensorflow/java/src/main/java/org/tensorflow/AbstractOperation.java
+87
View File
@@ -0,0 +1,87 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
/**
* Base class for {@link Operation} implementations.
*
* <p>As opposed to {@link Operation} itself, this class is package private and therefore its usage
* is limited to internal purposes only.
*/
abstract class AbstractOperation implements Operation {
@Override
public Output<?>[] outputList(int idx, int length) {
Output<?>[] outputs = new Output<?>[length];
for (int i = 0; i < length; ++i) {
outputs[i] = output(idx + i);
}
return outputs;
}
@Override
@SuppressWarnings({"rawtypes", "unchecked"})
public <T> Output<T> output(int idx) {
return new Output(this, idx);
}
@Override
public String toString() {
return String.format("<%s '%s'>", type(), name());
}
/**
* Returns the native handle of the {@code outputIdx}th output of this operation.
*
* <p>The nature of the returned value varies depending on current the execution environment.
*
* <ul>
* <li>In eager mode, the value is a handle to the tensor returned at this output.
* <li>In graph mode, the value is a handle to the operation itself, which should be paired with
* the index of the output when calling the native layer.
* </ul>
*
* @param outputIdx index of the output in this operation
* @return a native handle, see method description for more details
*/
abstract long getUnsafeNativeHandle(int outputIdx);
/**
* Returns the shape of the tensor of the {@code outputIdx}th output of this operation.
*
* @param outputIdx index of the output of this operation
* @return output tensor shape
*/
abstract long[] shape(int outputIdx);
/**
* Returns the datatype of the tensor of the {@code outputIdx}th output of this operation.
*
* @param outputIdx index of the output of this operation
* @return output tensor datatype
*/
abstract DataType dtype(int outputIdx);
/**
* Returns the tensor of the {@code outputIdx}th output of this operation.
*
* <p>This is only supported in an eager execution environment.
*
* @param outputIdx index of the output of this operation
* @return output tensor
*/
abstract Tensor<?> tensor(int outputIdx);
}
tensorflow/java/src/main/java/org/tensorflow/DataType.java
+116
View File
@@ -0,0 +1,116 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import java.util.HashMap;
import java.util.Map;
import org.tensorflow.types.UInt8;
/** Represents the type of elements in a {@link Tensor} as an enum. */
public enum DataType {
/** 32-bit single precision floating point. */
FLOAT(1, 4),
/** 64-bit double precision floating point. */
DOUBLE(2, 8),
/** 32-bit signed integer. */
INT32(3, 4),
/** 8-bit unsigned integer. */
UINT8(4, 1),
/**
* A sequence of bytes.
*
* <p>TensorFlow uses the STRING type for an arbitrary sequence of bytes.
*/
STRING(7, -1),
/** 64-bit signed integer. */
INT64(9, 8),
/** Boolean. */
BOOL(10, 1);
private final int value;
private final int byteSize;
/**
* @param value must match the corresponding TF_* value in the TensorFlow C API.
* @param byteSize size of an element of this type, in bytes, -1 if unknown
*/
DataType(int value, int byteSize) {
this.value = value;
this.byteSize = byteSize;
}
/**
* Returns the size of an element of this type, in bytes, or -1 if element size is variable.
*/
public int byteSize() {
return byteSize;
}
/** Corresponding value of the TF_DataType enum in the TensorFlow C API. */
int c() {
return value;
}
// Cached to avoid copying it
private static final DataType[] values = values();
static DataType fromC(int c) {
for (DataType t : values) {
if (t.value == c) {
return t;
}
}
throw new IllegalArgumentException(
"DataType " + c + " is not recognized in Java (version " + TensorFlow.version() + ")");
}
/**
* Returns the DataType of a Tensor whose elements have the type specified by class {@code c}.
*
* @param c The class describing the TensorFlow type of interest.
* @return The {@code DataType} enum corresponding to {@code c}.
* @throws IllegalArgumentException if objects of {@code c} do not correspond to a TensorFlow
* datatype.
*/
public static DataType fromClass(Class<?> c) {
DataType dtype = typeCodes.get(c);
if (dtype == null) {
throw new IllegalArgumentException(
c.getName() + " objects cannot be used as elements in a TensorFlow Tensor");
}
return dtype;
}
private static final Map<Class<?>, DataType> typeCodes = new HashMap<>();
static {
typeCodes.put(Float.class, DataType.FLOAT);
typeCodes.put(Double.class, DataType.DOUBLE);
typeCodes.put(Integer.class, DataType.INT32);
typeCodes.put(UInt8.class, DataType.UINT8);
typeCodes.put(Long.class, DataType.INT64);
typeCodes.put(Boolean.class, DataType.BOOL);
typeCodes.put(String.class, DataType.STRING);
}
}
tensorflow/java/src/main/java/org/tensorflow/EagerOperation.java
+172
View File
@@ -0,0 +1,172 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import java.util.concurrent.atomic.AtomicReferenceArray;
/**
* Implementation of an {@link Operation} executed eagerly.
*
* <p>EagerOperation instances are valid only as long as the {@link EagerSession} they are a part of
* is valid. Thus, if {@link EagerSession#close()} has been invoked, then methods on the
* EagerOperation instance may fail with an {@code IllegalStateException}.
*
* <p>EagerOperation instances are thread-safe.
*/
class EagerOperation extends AbstractOperation {
EagerOperation(
EagerSession session,
long opNativeHandle,
long[] outputNativeHandles,
String type,
String name) {
this.session = session;
this.type = type;
this.name = name;
this.nativeRef = new NativeReference(session, this, opNativeHandle, outputNativeHandles);
this.outputTensors = new AtomicReferenceArray<Tensor<?>>(outputNativeHandles.length);
}
@Override
public String name() {
return name;
}
@Override
public String type() {
return type;
}
@Override
public int numOutputs() {
return nativeRef.outputHandles.length;
}
@Override
public int outputListLength(final String name) {
return outputListLength(nativeRef.opHandle, name);
}
@Override
public int inputListLength(final String name) {
return inputListLength(nativeRef.opHandle, name);
}
@Override
public long getUnsafeNativeHandle(int outputIndex) {
return nativeRef.outputHandles[outputIndex];
}
@Override
public long[] shape(int outputIndex) {
// If the tensor of this output has already been resolved, return its shape.
// Otherwise, retrieve the tensor shape from the native library.
Tensor<?> tensor = outputTensors.get(outputIndex);
if (tensor != null) {
return tensor.shape();
}
long outputNativeHandle = getUnsafeNativeHandle(outputIndex);
long[] shape = new long[numDims(outputNativeHandle)];
for (int i = 0; i < shape.length; ++i) {
shape[i] = dim(outputNativeHandle, i);
}
return shape;
}
@Override
public DataType dtype(int outputIndex) {
// If the tensor of this output has already been resolved, return its datatype.
// Otherwise, retrieve the tensor datatype from the native library.
Tensor<?> tensor = outputTensors.get(outputIndex);
if (tensor != null) {
return tensor.dataType();
}
long outputNativeHandle = getUnsafeNativeHandle(outputIndex);
return DataType.fromC(dataType(outputNativeHandle));
}
@Override
public Tensor<?> tensor(int outputIndex) {
Tensor<?> tensor = outputTensors.get(outputIndex);
if (tensor == null) {
tensor = resolveTensor(outputIndex);
}
return tensor;
}
private final EagerSession session;
private final NativeReference nativeRef;
private final String type;
private final String name;
private final AtomicReferenceArray<Tensor<?>> outputTensors;
private Tensor<?> resolveTensor(int outputIndex) {
// Take an optimistic approach, where we attempt to resolve the output tensor without locking.
// If another thread has resolved it meanwhile, release our copy and reuse the existing one
// instead.
long tensorNativeHandle = resolveTensorHandle(getUnsafeNativeHandle(outputIndex));
Tensor<?> tensor = Tensor.fromHandle(tensorNativeHandle, session);
if (!outputTensors.compareAndSet(outputIndex, null, tensor)) {
tensor.close();
tensor = outputTensors.get(outputIndex);
}
return tensor;
}
private static class NativeReference extends EagerSession.NativeReference {
NativeReference(
EagerSession session, EagerOperation operation, long opHandle, long[] outputHandles) {
super(session, operation);
this.opHandle = opHandle;
this.outputHandles = outputHandles;
}
@Override
void delete() {
if (opHandle != 0L) {
for (int i = 0; i < outputHandles.length; ++i) {
if (outputHandles[i] != 0L) {
EagerOperation.deleteTensorHandle(outputHandles[i]);
outputHandles[i] = 0L;
}
}
EagerOperation.delete(opHandle);
opHandle = 0L;
}
}
private long opHandle;
private final long[] outputHandles;
}
private static native void delete(long handle);
private static native void deleteTensorHandle(long handle);
private static native long resolveTensorHandle(long handle);
private static native int outputListLength(long handle, String name);
private static native int inputListLength(long handle, String name);
private static native int dataType(long handle);
private static native int numDims(long handle);
private static native long dim(long handle, int index);
}
tensorflow/java/src/main/java/org/tensorflow/EagerOperationBuilder.java
+258
View File
@@ -0,0 +1,258 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
/**
* An {@link OperationBuilder} for building {@link Operation Operations} that are executed eagerly.
*/
final class EagerOperationBuilder implements OperationBuilder {
EagerOperationBuilder(EagerSession session, String type, String name) {
this.session = session;
this.type = type;
this.name = name;
this.nativeRef = new NativeReference(session, this, allocate(session.nativeHandle(), type));
}
@Override
public EagerOperation build() {
long[] tensorHandles = execute(nativeRef.opHandle);
EagerOperation operation =
new EagerOperation(session, nativeRef.opHandle, tensorHandles, type, name);
// Release our reference to the native op handle now that we transferred its
// ownership to the EagerOperation
nativeRef.clear();
return operation;
}
@Override
public EagerOperationBuilder addInput(Output<?> input) {
addInput(nativeRef.opHandle, input.getUnsafeNativeHandle());
return this;
}
@Override
public EagerOperationBuilder addInputList(Output<?>[] inputs) {
long[] inputHandles = new long[inputs.length];
for (int i = 0; i < inputs.length; ++i) {
inputHandles[i] = inputs[i].getUnsafeNativeHandle();
}
addInputList(nativeRef.opHandle, inputHandles);
return this;
}
@Override
public OperationBuilder addControlInput(Operation control) {
throw new UnsupportedOperationException(
"Control inputs are not supported in an eager execution environment");
}
@Override
public EagerOperationBuilder setDevice(String device) {
setDevice(nativeRef.opHandle, device);
return this;
}
@Override
public EagerOperationBuilder setAttr(String name, String value) {
return setAttr(name, value.getBytes(StandardCharsets.UTF_8));
}
@Override
public EagerOperationBuilder setAttr(String name, String[] values) {
Charset utf8 = StandardCharsets.UTF_8;
Object[] objects = new Object[values.length];
for (int i = 0; i < values.length; ++i) {
objects[i] = values[i].getBytes(utf8);
}
setAttrStringList(nativeRef.opHandle, name, values);
return this;
}
@Override
public EagerOperationBuilder setAttr(String name, byte[] values) {
setAttrString(nativeRef.opHandle, name, values);
return this;
}
@Override
public EagerOperationBuilder setAttr(String name, long value) {
setAttrInt(nativeRef.opHandle, name, value);
return this;
}
@Override
public EagerOperationBuilder setAttr(String name, long[] values) {
setAttrIntList(nativeRef.opHandle, name, values);
return this;
}
@Override
public EagerOperationBuilder setAttr(String name, float value) {
setAttrFloat(nativeRef.opHandle, name, value);
return this;
}
@Override
public EagerOperationBuilder setAttr(String name, float[] values) {
setAttrFloatList(nativeRef.opHandle, name, values);
return this;
}
@Override
public EagerOperationBuilder setAttr(String name, boolean value) {
setAttrBool(nativeRef.opHandle, name, value);
return this;
}
@Override
public EagerOperationBuilder setAttr(String name, boolean[] values) {
setAttrBoolList(nativeRef.opHandle, name, values);
return this;
}
@Override
public EagerOperationBuilder setAttr(String name, DataType value) {
setAttrType(nativeRef.opHandle, name, value.c());
return this;
}
@Override
public EagerOperationBuilder setAttr(String name, DataType[] values) {
int[] c = new int[values.length];
for (int i = 0; i < values.length; ++i) {
c[i] = values[i].c();
}
setAttrTypeList(nativeRef.opHandle, name, c);
return this;
}
@Override
public EagerOperationBuilder setAttr(String name, Tensor<?> value) {
setAttrTensor(nativeRef.opHandle, name, value.getNativeHandle());
return this;
}
@Override
public EagerOperationBuilder setAttr(String name, Tensor<?>[] values) {
// TODO (karllessard) could be supported by adding this attribute type in the eager C API
throw new UnsupportedOperationException(
"Tensor list attributes are not supported in eager mode");
}
@Override
public EagerOperationBuilder setAttr(String name, Shape value) {
setAttrShape(nativeRef.opHandle, name, value.asArray(), value.numDimensions());
return this;
}
@Override
public EagerOperationBuilder setAttr(String name, Shape[] values) {
int[] numDimensions = new int[values.length];
int totalNumDimensions = 0;
for (int idx = 0; idx < values.length; ++idx) {
int n = values[idx].numDimensions();
numDimensions[idx] = n;
if (n > 0) {
totalNumDimensions += n;
}
}
// Flatten the shapes into a single array to avoid too much overhead in the
// native part
long[] shapes = new long[totalNumDimensions];
int shapeIdx = 0;
for (Shape shape : values) {
if (shape.numDimensions() > 0) {
for (long dim : shape.asArray()) {
shapes[shapeIdx++] = dim;
}
}
}
setAttrShapeList(nativeRef.opHandle, name, shapes, numDimensions);
return this;
}
private static class NativeReference extends EagerSession.NativeReference {
NativeReference(EagerSession session, EagerOperationBuilder operation, long opHandle) {
super(session, operation);
this.opHandle = opHandle;
}
@Override
public void clear() {
super.clear();
opHandle = 0L;
}
@Override
synchronized void delete() {
if (opHandle != 0L) {
EagerOperationBuilder.delete(opHandle);
opHandle = 0L;
}
}
private long opHandle;
}
private final EagerSession session;
private final String type;
private final String name;
private final NativeReference nativeRef;
private static native long allocate(long ctxHandle, String type);
private static native void delete(long opHandle);
private static native long[] execute(long opHandle);
private static native void addInput(long opHandle, long tensorHandle);
private static native void addInputList(long opHandle, long[] tensorHandles);
private static native void setDevice(long opHandle, String device);
private static native void setAttrString(long opHandle, String name, byte[] value);
private static native void setAttrStringList(long opHandle, String name, Object[] value);
private static native void setAttrInt(long opHandle, String name, long value);
private static native void setAttrIntList(long opHandle, String name, long[] values);
private static native void setAttrFloat(long opHandle, String name, float value);
private static native void setAttrFloatList(long opHandle, String name, float[] values);
private static native void setAttrBool(long opHandle, String name, boolean value);
private static native void setAttrBoolList(long opHandle, String name, boolean[] values);
private static native void setAttrType(long opHandle, String name, int type);
private static native void setAttrTypeList(long opHandle, String name, int[] types);
private static native void setAttrTensor(long opHandle, String name, long tensorHandle);
private static native void setAttrShape(long opHandle, String name, long[] shape, int numDims);
private static native void setAttrShapeList(
long opHandle, String name, long[] shapes, int[] numDims);
}
tensorflow/java/src/main/java/org/tensorflow/EagerSession.java
+533
View File
@@ -0,0 +1,533 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import java.lang.ref.PhantomReference;
import java.lang.ref.Reference;
import java.lang.ref.ReferenceQueue;
import java.util.IdentityHashMap;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* An environment for executing TensorFlow operations eagerly.
*
* <p>Eager execution is an imperative programming environment that evaluates operations
* immediately, without building graphs. Operations return concrete values instead of constructing a
* computational graph to run later, as with {@link Graph}s and {@link Session}s.
*
* <p>This makes it easy to develop with TensorFlow and debug models, as it behaves more like a
* standard programming library.
*
* <p>Instances of a {@code EagerSession} are thread-safe.
*/
public final class EagerSession implements ExecutionEnvironment, AutoCloseable {
/**
* Controls how to act when we try to run an operation on a given device but some input tensors
* are not on that device.
*/
public static enum DevicePlacementPolicy {
/** Running operations with input tensors on the wrong device will fail. */
EXPLICIT(0),
/** Copy the tensor to the right device but log a warning. */
WARN(1),
/**
* Silently copy the tensor, which has a performance cost since the operation will be blocked
* till the copy completes. This is the default placement policy.
*/
SILENT(2),
/** Placement policy which silently copies int32 tensors but not other dtypes. */
SILENT_FOR_INT32(3);
private DevicePlacementPolicy(int code) {
this.code = code;
}
private final int code;
}
/**
* Controls how TensorFlow resources are cleaned up when they are no longer needed.
*
* <p>All resources allocated during an {@code EagerSession} are deleted when the session is
* closed. To prevent out-of-memory errors, it is also strongly suggest to cleanup those resources
* during the session. For example, executing n operations in a loop of m iterations will allocate
* a minimum of n*m resources while in most cases, only resources of the last iteration are still
* being used.
*
* <p>{@code EagerSession} instances can be notified in different ways when TensorFlow objects are
* no longer being referred, so they can proceed to the cleanup of any resources they owned.
*/
public static enum ResourceCleanupStrategy {
/**
* Monitor and delete unused resources from a new thread running in background.
*
* <p>This is the most reliable approach to cleanup TensorFlow resources, at the cost of
* starting and running an additional thread dedicated to this task. Each {@code EagerSession}
* instance has its own thread, which is stopped only when the session is closed.
*
* <p>This strategy is used by default.
*/
IN_BACKGROUND,
/**
* Monitor and delete unused resources from existing threads, before or after they complete
* another task.
*
* <p>Unused resources are released when a call to the TensorFlow library reaches a safe point
* for cleanup. This is done synchronously and might block for a short period of time the thread
* who triggered that call.
*
* <p>This strategy should be used only if, for some reasons, no additional thread should be
* allocated for cleanup. Otherwise, {@link #IN_BACKGROUND} should be preferred.
*/
ON_SAFE_POINTS,
/**
* Only delete resources when the session is closed.
*
* <p>All resources allocated during the session will remained in memory until the session is
* explicitly closed (or via the traditional `try-with-resource` technique). No extra task for
* resource cleanup will be attempted.
*
* <p>This strategy can lead up to out-of-memory errors and its usage is not recommended, unless
* the scope of the session is limited to execute only a small amount of operations.
*/
ON_SESSION_CLOSE,
}
public static class Options {
/**
* Controls how operations dispatched are actually executed.
*
* <p>When set to true, each operation are executed asynchronously (in which case some
* operations might return "non-ready" outputs). When set to false, all operations are executed
* synchronously.
*
* <p>Synchronous execution is used by default.
*
* @param value true for asynchronous execution, false for synchronous.
*/
public Options async(boolean value) {
async = value;
return this;
}
/**
* Controls how to act when we try to run an operation on a given device but some input tensors
* are not on that device.
*
* <p>{@link DevicePlacementPolicy#SILENT} is used by default.
*
* @param value policy to apply
* @see DevicePlacementPolicy
*/
public Options devicePlacementPolicy(DevicePlacementPolicy value) {
devicePlacementPolicy = value;
return this;
}
/**
* Controls how TensorFlow resources are cleaned up when no longer needed.
*
* <p>{@link ResourceCleanupStrategy#IN_BACKGROUND} is used by default.
*
* @param value strategy to use
* @see ResourceCleanupStrategy
*/
public Options resourceCleanupStrategy(ResourceCleanupStrategy value) {
resourceCleanupStrategy = value;
return this;
}
/**
* Configures the session based on the data found in the provided buffer, which is serialized
* TensorFlow config proto.
*
* <p>Warning: the support of this feature is subject to changes since TensorFlow protos might
* not be supported on public endpoints in the future.
*
* <p>See also: <a href="https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/config.proto">config.proto</a>
*
* @param value a serialized config proto
*/
public Options config(byte[] value) {
config = value;
return this;
}
/** Builds an eager session with the selected options. */
public EagerSession build() {
return new EagerSession(this, new ReferenceQueue<Object>());
}
// For garbage-collection tests only
EagerSession buildForGcTest(ReferenceQueue<Object> gcQueue) {
return new EagerSession(this, gcQueue);
}
private boolean async;
private DevicePlacementPolicy devicePlacementPolicy;
private ResourceCleanupStrategy resourceCleanupStrategy;
private byte[] config;
private Options() {
async = false;
devicePlacementPolicy = DevicePlacementPolicy.SILENT;
resourceCleanupStrategy = ResourceCleanupStrategy.IN_BACKGROUND;
config = null;
}
}
/**
* Initializes the default eager session, which remains active for the lifetime of the
* application.
*
* <p>This method is implicitly invoked on the first call to {@link #getDefault()}, but can also
* be invoked explicitly to override default options.
*
* <p>Note that calling this method more than once will throw an {@code IllegalArgumentException}
* as the default session cannot be modified once it has been created. Therefore, it is important
* to explicitly initialize it before {@link #getDefault()} is invoked for the first time from any
* thread.
*
* <p>Example usage:
*
* <pre>{@code
* // Initializing default session to override default options is valid but
* // is optional
* EagerSession.initDefault(EagerSession.options().async(true));
*
* // Starting to build eager operations using default session, by calling
* // EagerSession.getDefault() implicitly
* Ops tf = Ops.create();
*
* // Initializing default session more than once or after using it is not
* // permitted and throws an exception
* EagerSession.initDefault(EagerSession.options().async(true)); // throws
* }</pre>
*
* @param options options to use to build default session
* @return default eager session
* @throws IllegalStateException if the default session is already initialized
* @see #getDefault()
*/
public static EagerSession initDefault(Options options) {
synchronized (EagerSession.class) {
if (defaultSession != null) {
throw new IllegalStateException("Default eager session is already initialized");
}
defaultSession = options.build();
}
return defaultSession;
}
/**
* Returns the default eager session
*
* <p>Once initialized, the default eager session remains active for the whole life of the
* application, as opposed to sessions obtained from {@link #create()} or {@link Options#build()}
* which should be closed after their usage.
*
* <p>The default set of {@link Options} is used to initialize the session on the first call. To
* override this behavior, it is possible to invoke {@link #initDefault(Options)} with a different
* set of options prior to this first call.
*
* <p>Example usage:
*
* <pre>{@code
* // Starting to build eager operations using default session, by calling
* // EagerSession.getDefault() implicitly
* Ops tf = Ops.create();
*
* // Starting to build eager operations using default session, by calling
* // EagerSession.getDefault() explicitly
* Ops tf = Ops.create(EagerSession.getDefault());
* }</pre>
*
* @return default eager session
* @see #initDefault
*/
public static EagerSession getDefault() {
if (defaultSession == null) {
synchronized (EagerSession.class) {
if (defaultSession == null) {
defaultSession = options().build();
}
}
}
return defaultSession;
}
/**
* Returns an {@code EagerSession} configured with default options.
*
* <p><b>WARNING:</b>Instances of {@code EagerSession} returned by this method must be explicitly
* freed by invoking {@link #close()} when they are no longer needed. This could be achieve using
* the `try-with-resources` technique.
*
* <p>Example usage:
*
* <pre>{@code
* try (EagerSession session = EagerSession.create()) {
* Ops tf = Ops.create(session);
* // build execute operations eagerly...
* }
* }</pre>
*/
public static EagerSession create() {
return options().build();
}
/**
* Returns an object that configures and builds a {@code EagerSession} with custom options.
*
* <p><b>WARNING:</b>Instances of {@code EagerSession} returned by this method must be explicitly
* freed by invoking {@link #close()} when they are no longer needed. This could be achieve using
* the `try-with-resources` technique.
*
* <p>Example usage:
*
* <pre>{@code
* try (EagerSession session = EagerSession.options().async(true).build()) {
* Ops tf = Ops.create(session);
* // build execute operations eagerly and asynchronously...
* }
* }</pre>
*/
public static EagerSession.Options options() {
return new Options();
}
@Override
public synchronized void close() {
if (this == defaultSession) {
throw new IllegalStateException("Default eager session cannot be closed");
}
if (nativeHandle != 0L) {
if (resourceCleanupStrategy == ResourceCleanupStrategy.IN_BACKGROUND) {
nativeResources.stopCleanupThread();
}
nativeResources.deleteAll();
delete(nativeHandle);
nativeHandle = 0L;
}
}
@Override
public OperationBuilder opBuilder(String type, String name) {
if (resourceCleanupStrategy == ResourceCleanupStrategy.ON_SAFE_POINTS) {
nativeResources.tryCleanup();
}
checkSession();
return new EagerOperationBuilder(this, type, name);
}
long nativeHandle() {
checkSession();
return nativeHandle;
}
ResourceCleanupStrategy resourceCleanupStrategy() {
return resourceCleanupStrategy;
}
/**
* A reference to one or more allocated native resources.
*
* <p>Any Java objects owning native resources must declare a reference to those resources in a
* subclass that extends from {@code NativeReference}. When {@link NativeReference#delete()} is
* invoked, the resources must be freed. For example:
*
* <pre>{@code
* private static class NativeReference extends EagerSession.NativeReference {
*
* NativeReference(EagerSession session, MyClass referent, long handle) {
* super(session, referent);
* this.handle = handle;
* }
*
* @Override
* void delete() {
* MyClass.nativeDelete(handle);
* }
*
* private final long handle;
* }
* }</pre>
*
* A Java object "owns" a native resource if this resource should not survive beyond the lifetime
* of this object.
*
* <p><b>IMPORTANT</b>: All nested subclasses of {@code NativeReference} must be declared as
* static, otherwise their instances will hold an implicit reference to their enclosing object,
* preventing the garbage collector to release them when they are no longer needed.
*/
abstract static class NativeReference extends PhantomReference<Object> {
/** Attach a new phantom reference of {@code referent} to {@code session}. */
public NativeReference(EagerSession session, Object referent) {
super(referent, session.nativeResources.garbageQueue);
session.checkSession();
nativeResources = session.nativeResources;
nativeResources.attach(this);
}
/**
* Detach this reference from its current session.
*
* <p>Clearing a NativeReference does not invoke {@link #delete()}, thus won't release the
* native resources it refers to. It can be used when passing the ownership of those resources
* to another object.
*
* <p>If native resources needs to be deleted as well, call {@link #delete()} explicitly.
*/
@Override
public void clear() {
nativeResources.detach(this);
super.clear();
}
/** Releases all native resources owned by the referred object, now deleted. */
abstract void delete();
private final NativeResourceCollector nativeResources;
}
/**
* Collects native references attached to this session and releases their resources if they are no
* longer needed.
*/
private static class NativeResourceCollector {
NativeResourceCollector(ReferenceQueue<Object> garbageQueue) {
this.garbageQueue = garbageQueue;
}
void attach(NativeReference nativeRef) {
synchronized (nativeRefs) {
nativeRefs.put(nativeRef, null);
}
}
void detach(NativeReference nativeRef) {
synchronized (nativeRefs) {
nativeRefs.remove(nativeRef);
}
}
void delete(NativeReference nativeRef) {
synchronized (nativeRefs) {
if (!nativeRefs.keySet().remove(nativeRef)) {
return; // safety check
}
}
nativeRef.delete();
}
void deleteAll() {
synchronized (nativeRefs) {
for (NativeReference nativeRef : nativeRefs.keySet()) {
nativeRef.delete();
}
nativeRefs.clear();
}
}
void tryCleanup() {
Reference<?> nativeRef;
synchronized (nativeRefs) {
while ((nativeRef = garbageQueue.poll()) != null) {
delete((NativeReference) nativeRef);
}
}
}
synchronized void startCleanupThread() {
if (cleanupInBackground) {
return; // ignore if cleanup thread is already running
}
try {
cleanupInBackground = true;
cleanupService.execute(
new Runnable() {
@Override
public void run() {
try {
while (cleanupInBackground) {
NativeReference nativeRef = (NativeReference) garbageQueue.remove();
delete(nativeRef);
}
} catch (InterruptedException e) {
// exit
}
}
});
} catch (Exception e) {
cleanupInBackground = false;
throw e;
}
}
void stopCleanupThread() {
cleanupInBackground = false;
cleanupService.shutdownNow(); // returns without waiting for the thread to stop
}
private final ExecutorService cleanupService = Executors.newSingleThreadExecutor();
private final Map<NativeReference, Void> nativeRefs = new IdentityHashMap<>();
private final ReferenceQueue<Object> garbageQueue;
private volatile boolean cleanupInBackground = false;
}
private static volatile EagerSession defaultSession = null;
private final NativeResourceCollector nativeResources;
private final ResourceCleanupStrategy resourceCleanupStrategy;
private long nativeHandle;
private EagerSession(Options options, ReferenceQueue<Object> garbageQueue) {
this.nativeResources = new NativeResourceCollector(garbageQueue);
this.nativeHandle = allocate(options.async, options.devicePlacementPolicy.code, options.config);
this.resourceCleanupStrategy = options.resourceCleanupStrategy;
if (resourceCleanupStrategy == ResourceCleanupStrategy.IN_BACKGROUND) {
nativeResources.startCleanupThread();
}
}
private void checkSession() {
if (nativeHandle == 0L) {
throw new IllegalStateException("Eager session has been closed");
}
}
private static native long allocate(boolean async, int devicePlacementPolicy, byte[] config);
private static native void delete(long handle);
static {
TensorFlow.init();
}
}
tensorflow/java/src/main/java/org/tensorflow/ExecutionEnvironment.java
+31
View File
@@ -0,0 +1,31 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
/** Defines an environment for creating and executing TensorFlow {@link Operation}s. */
public interface ExecutionEnvironment {
/**
* Returns a builder to create a new {@link Operation}.
*
* @param type of the Operation (i.e., identifies the computation to be performed)
* @param name to refer to the created Operation in this environment scope.
* @return an {@link OperationBuilder} to create an Operation when {@link
* OperationBuilder#build()} is invoked. If {@link OperationBuilder#build()} is not invoked,
* then some resources may leak.
*/
OperationBuilder opBuilder(String type, String name);
}
tensorflow/java/src/main/java/org/tensorflow/Graph.java
+481
View File
@@ -0,0 +1,481 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import java.util.Iterator;
/**
* A data flow graph representing a TensorFlow computation.
*
* <p>Instances of a Graph are thread-safe.
*
* <p><b>WARNING:</b> Resources consumed by the Graph object must be explicitly freed by invoking
* the {@link #close()} method then the Graph object is no longer needed.
*/
public final class Graph implements ExecutionEnvironment, AutoCloseable {
/** Create an empty Graph. */
public Graph() {
nativeHandle = allocate();
}
/** Create a Graph from an existing handle (takes ownership). */
Graph(long nativeHandle) {
this.nativeHandle = nativeHandle;
}
/**
* Release resources associated with the Graph.
*
* <p>Blocks until there are no active {@link Session} instances referring to this Graph. A Graph
* is not usable after close returns.
*/
@Override
public void close() {
synchronized (nativeHandleLock) {
if (nativeHandle == 0) {
return;
}
while (refcount > 0) {
try {
nativeHandleLock.wait();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
// Possible leak of the graph in this case?
return;
}
}
delete(nativeHandle);
nativeHandle = 0;
}
}
/**
* Returns the operation (node in the Graph) with the provided name.
*
* <p>Or {@code null} if no such operation exists in the Graph.
*/
public GraphOperation operation(String name) {
synchronized (nativeHandleLock) {
long oph = operation(nativeHandle, name);
if (oph == 0) {
return null;
}
return new GraphOperation(this, oph);
}
}
/**
* Iterator over all the {@link Operation}s in the graph.
*
* <p>The order of iteration is unspecified. Consumers of the iterator will receive no
* notification should the underlying graph change during iteration.
*/
public Iterator<Operation> operations() {
return new OperationIterator(this);
}
/**
* Returns a builder to add {@link Operation}s to the Graph.
*
* @param type of the Operation (i.e., identifies the computation to be performed)
* @param name to refer to the created Operation in the graph.
* @return an {@link OperationBuilder}, which will add the Operation to the graph when {@link
* OperationBuilder#build()} is invoked. If {@link OperationBuilder#build()} is not invoked,
* then some resources may leak.
*/
@Override
public GraphOperationBuilder opBuilder(String type, String name) {
return new GraphOperationBuilder(this, type, name);
}
/**
* Import a serialized representation of a TensorFlow graph.
*
* <p>The serialized representation of the graph, often referred to as a <i>GraphDef</i>, can be
* generated by {@link #toGraphDef()} and equivalents in other language APIs.
*
* @throws IllegalArgumentException if graphDef is not a recognized serialization of a graph.
* @see #importGraphDef(byte[], String)
*/
public void importGraphDef(byte[] graphDef) throws IllegalArgumentException {
importGraphDef(graphDef, "");
}
/**
* Import a serialized representation of a TensorFlow graph.
*
* @param graphDef the serialized representation of a TensorFlow graph.
* @param prefix a prefix that will be prepended to names in graphDef
* @throws IllegalArgumentException if graphDef is not a recognized serialization of a graph.
* @see #importGraphDef(byte[])
*/
public void importGraphDef(byte[] graphDef, String prefix) throws IllegalArgumentException {
if (graphDef == null || prefix == null) {
throw new IllegalArgumentException("graphDef and prefix cannot be null");
}
synchronized (nativeHandleLock) {
importGraphDef(nativeHandle, graphDef, prefix);
}
}
/**
* Generate a serialized representation of the Graph.
*
* @see #importGraphDef(byte[])
* @see #importGraphDef(byte[], String)
*/
public byte[] toGraphDef() {
synchronized (nativeHandleLock) {
return toGraphDef(nativeHandle);
}
}
/**
* Adds operations to compute the partial derivatives of sum of {@code y}s w.r.t {@code x}s, i.e.,
* {@code d(y_1 + y_2 + ...)/dx_1, d(y_1 + y_2 + ...)/dx_2...}
*
* <p>{@code dx} are used as initial gradients (which represent the symbolic partial derivatives
* of some loss function {@code L} w.r.t. {@code y}). {@code dx} must be null or have size of
* {@code y}.
*
* <p>If {@code dx} is null, the implementation will use dx of {@link
* org.tensorflow.op.core.OnesLike OnesLike} for all shapes in {@code y}.
*
* <p>{@code prefix} is used as the name prefix applied to all nodes added to the graph to compute
* gradients. It must be unique within the provided graph or the operation will fail.
*
* <p>If {@code prefix} is null, then one will be chosen automatically.
*
* @param prefix unique string prefix applied before the names of nodes added to the graph to
* compute gradients. If null, a default one will be chosen.
* @param y output of the function to derive
* @param x inputs of the function for which partial derivatives are computed
* @param dx if not null, the partial derivatives of some loss function {@code L} w.r.t. {@code y}
* @return the partial derivatives {@code dy} with the size of {@code x}
*/
public Output<?>[] addGradients(String prefix, Output<?>[] y, Output<?>[] x, Output<?>[] dx) {
Output<?>[] dy = new Output<?>[x.length];
final long[] yHandles = new long[y.length];
final int[] yIndices = new int[y.length];
final long[] xHandles = new long[x.length];
final int[] xIndices = new int[x.length];
long[] dxHandles = null;
int[] dxIndices = null;
try (Reference ref = ref()) {
for (int i = 0; i < y.length; ++i) {
yHandles[i] = y[i].getUnsafeNativeHandle();
yIndices[i] = y[i].index();
}
for (int i = 0; i < x.length; ++i) {
xHandles[i] = x[i].getUnsafeNativeHandle();
xIndices[i] = x[i].index();
}
if (dx != null && dx.length > 0) {
dxHandles = new long[dx.length];
dxIndices = new int[dx.length];
for (int i = 0; i < dx.length; ++i) {
dxHandles[i] = dx[i].getUnsafeNativeHandle();
dxIndices[i] = dx[i].index();
}
}
// Gradient outputs are returned in two continuous arrays concatenated into one. The first
// holds the native handles of the gradient operations while the second holds the index of
// their output e.g. given
// xHandles = [x0Handle, x1Handle, ...] and xIndices = [x0Index, x1Index, ..], we obtain
// dy = [dy0Handle, dy1Handle, ..., dy0Index, dy1Index, ...]
long[] dyHandlesAndIndices =
addGradients(
ref.nativeHandle(),
prefix,
yHandles,
yIndices,
xHandles,
xIndices,
dxHandles,
dxIndices);
int ndy = dyHandlesAndIndices.length >> 1;
if (ndy != dy.length) {
throw new IllegalStateException(String.valueOf(ndy) + " gradients were added to the graph when " + dy.length
+ " were expected");
}
for (int i = 0, j = ndy; i < ndy; ++i, ++j) {
GraphOperation op = new GraphOperation(this, dyHandlesAndIndices[i]);
dy[i] = new Output<>(op, (int) dyHandlesAndIndices[j]);
}
}
return dy;
}
/**
* Adds operations to compute the partial derivatives of sum of {@code y}s w.r.t {@code x}s,
* i.e., {@code dy/dx_1, dy/dx_2...}
* <p>
* This is a simplified version of {@link #addGradients(String, Output[], Output[], Output[])}
* where {@code y} is a single output, {@code dx} is null and {@code prefix} is null.
*
* @param y output of the function to derive
* @param x inputs of the function for which partial derivatives are computed
* @return the partial derivatives {@code dy} with the size of {@code x}
*/
public Output<?>[] addGradients(Output<?> y, Output<?>[] x) {
return addGradients(null, new Output<?>[] {y}, x, null);
}
/**
* Used to instantiate an abstract class which overrides the buildSubgraph method to build a
* conditional or body subgraph for a while loop. After Java 8, this can alternatively be used to
* create a lambda for the same purpose.
*
* <p>To be used when calling {@link #whileLoop(Output[],
* org.tensorflow.Graph.WhileSubgraphBuilder, org.tensorflow.Graph.WhileSubgraphBuilder, String)}
*
* <p>Example usage (prior to Java 8):
*
* <p>{@code WhileSubgraphBuilder bodyGraphBuilder = new WhileSubgraphBuilder() { @Override public
* void buildSubgraph(Graph bodyGraph, Output<?>[] bodyInputs, Output<?>[] bodyOutputs) { // build
* body subgraph } }; }
*
* <p>Example usage (after Java 8):
*
* <p>{@code WhileSubgraphBuilder bodyGraphBuilder = (bodyGraph, bodyInputs, bodyOutputs) -> { //
* build body subgraph };}
*/
public interface WhileSubgraphBuilder {
/**
* To be overridden by user with code to build conditional or body subgraph for a while loop
*
* @param g the subgraph
* @param inputs subgraph inputs
* @param outputs subgraph outputs
*/
public void buildSubgraph(Graph g, Output<?>[] inputs, Output<?>[] outputs);
}
// called by while loop code in graph_jni.cc to construct conditional/body subgraphs
private static long[] buildSubgraph(
WhileSubgraphBuilder subgraphBuilder,
long subgraphHandle,
long[] inputHandles,
int[] inputIndices,
long[] outputHandles,
int[] outputIndices) {
Graph subgraph = new Graph(subgraphHandle);
int ninputs = inputHandles.length;
int noutputs = outputHandles.length;
Output<?>[] inputs = new Output<?>[ninputs];
Output<?>[] outputs = new Output<?>[noutputs];
long[] outputHandlesAndIndices = new long[noutputs * 2];
synchronized (subgraph.nativeHandleLock) {
try (Reference ref = subgraph.ref()) {
for (int i = 0; i < ninputs; i++) {
Operation op = new GraphOperation(subgraph, inputHandles[i]);
inputs[i] = op.output(inputIndices[i]);
}
for (int i = 0; i < noutputs; i++) {
Operation op = new GraphOperation(subgraph, outputHandles[i]);
outputs[i] = op.output(outputIndices[i]);
}
subgraphBuilder.buildSubgraph(subgraph, inputs, outputs);
for (int i = 0, j = noutputs; i < noutputs; i++, j++) {
outputHandlesAndIndices[i] = outputs[i].getUnsafeNativeHandle();
outputHandlesAndIndices[j] = (long) outputs[i].index();
}
}
return outputHandlesAndIndices;
}
}
/**
* Builds a while loop.
*
* @param inputs the loop inputs
* @param cgBuilder WhileSubgraphBuilder to build the conditional subgraph
* @param bgBuilder WhileSubgraphBuilder to build the body subgraph
* @param name name for the loop
* @return list of loop outputs, of the same length as {@code inputs}
*/
public Output<?>[] whileLoop(
Output<?>[] inputs,
WhileSubgraphBuilder cgBuilder,
WhileSubgraphBuilder bgBuilder,
String name) {
int ninputs = inputs.length;
long[] inputHandles = new long[ninputs];
int[] inputIndices = new int[ninputs];
Output<?>[] outputs = new Output<?>[ninputs];
synchronized (nativeHandleLock) {
try (Reference ref = ref()) {
for (int i = 0; i < ninputs; i++) {
inputHandles[i] = inputs[i].getUnsafeNativeHandle();
inputIndices[i] = inputs[i].index();
}
long[] outputHandlesAndIndices =
whileLoop(nativeHandle, inputHandles, inputIndices, name, cgBuilder, bgBuilder);
for (int i = 0, j = ninputs; i < ninputs; ++i, ++j) {
Operation op = new GraphOperation(this, outputHandlesAndIndices[i]);
outputs[i] = op.output((int) outputHandlesAndIndices[j]);
}
}
return outputs;
}
}
private final Object nativeHandleLock = new Object();
private long nativeHandle;
private int refcount = 0;
// Related native objects (such as the TF_Operation object backing an Operation instance)
// have a validity tied to that of the Graph. The handles to those native objects are not
// valid after Graph.close() has been invoked.
//
// Instances of the Reference class should be used to ensure the Graph has not been closed
// while dependent handles are in use.
class Reference implements AutoCloseable {
private Reference() {
synchronized (Graph.this.nativeHandleLock) {
active = Graph.this.nativeHandle != 0;
if (!active) {
throw new IllegalStateException("close() has been called on the Graph");
}
active = true;
Graph.this.refcount++;
}
}
@Override
public void close() {
synchronized (Graph.this.nativeHandleLock) {
if (!active) {
return;
}
active = false;
if (--Graph.this.refcount == 0) {
Graph.this.nativeHandleLock.notifyAll();
}
}
}
public long nativeHandle() {
synchronized (Graph.this.nativeHandleLock) {
return active ? Graph.this.nativeHandle : 0;
}
}
private boolean active;
}
Reference ref() {
return new Reference();
}
private static final class OperationIterator implements Iterator<Operation> {
OperationIterator(Graph g) {
this.graph = g;
this.operation = null;
this.position = 0;
this.advance();
}
private final void advance() {
Graph.Reference reference = this.graph.ref();
this.operation = null;
try {
long[] nativeReturn = nextOperation(reference.nativeHandle(), this.position);
if ((nativeReturn != null) && (nativeReturn[0] != 0)) {
this.operation = new GraphOperation(this.graph, nativeReturn[0]);
this.position = (int) nativeReturn[1];
}
} finally {
reference.close();
}
}
@Override
public boolean hasNext() {
return (this.operation != null);
}
@Override
public Operation next() {
Operation rhett = this.operation;
this.advance();
return rhett;
}
@Override
public void remove() {
throw new UnsupportedOperationException("remove() is unsupported.");
}
private final Graph graph;
private Operation operation;
private int position;
}
private static native long allocate();
private static native void delete(long handle);
private static native long operation(long handle, String name);
// This method returns the Operation native handle at index 0 and the new value for pos at index 1
// (see TF_GraphNextOperation)
private static native long[] nextOperation(long handle, int position);
private static native void importGraphDef(long handle, byte[] graphDef, String prefix)
throws IllegalArgumentException;
private static native byte[] toGraphDef(long handle);
private static native long[] addGradients(
long handle,
String prefix,
long[] inputHandles,
int[] inputIndices,
long[] outputHandles,
int[] outputIndices,
long[] gradInputHandles,
int[] gradInputIndices);
private static native long[] whileLoop(
long handle,
long[] inputHandles,
int[] inputIndices,
String name,
WhileSubgraphBuilder condGraphBuilder,
WhileSubgraphBuilder bodyGraphBuilder);
static {
TensorFlow.init();
}
}
tensorflow/java/src/main/java/org/tensorflow/GraphOperation.java
+168
View File
@@ -0,0 +1,168 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
/**
* Implementation for an {@link Operation} added as a node to a {@link Graph}.
*
* <p>GraphOperation instances are valid only as long as the {@link Graph} they are a part of is
* valid. Thus, if {@link Graph#close()} has been invoked, then methods on the GraphOperation
* instance may fail with an {@code IllegalStateException}.
*
* <p>GraphOperation instances are immutable and thread-safe.
*/
public final class GraphOperation extends AbstractOperation {
// Create an GraphOperation instance referring to an operation in g, with the given handle to the
// C
// TF_Operation object. The handle is valid only as long as g has not been closed, hence it is
// called unsafeHandle. Graph.ref() is used to safely use the unsafeHandle.
GraphOperation(Graph g, long unsafeNativeHandle) {
this.graph = g;
this.unsafeNativeHandle = unsafeNativeHandle;
}
@Override
public String name() {
Graph.Reference r = graph.ref();
try {
return name(getUnsafeNativeHandle());
} finally {
r.close();
}
}
@Override
public String type() {
Graph.Reference r = graph.ref();
try {
return type(getUnsafeNativeHandle());
} finally {
r.close();
}
}
@Override
public int numOutputs() {
Graph.Reference r = graph.ref();
try {
return numOutputs(getUnsafeNativeHandle());
} finally {
r.close();
}
}
@Override
public int outputListLength(final String name) {
Graph.Reference r = graph.ref();
try {
return outputListLength(getUnsafeNativeHandle(), name);
} finally {
r.close();
}
}
@Override
public int hashCode() {
return Long.valueOf(getUnsafeNativeHandle()).hashCode();
}
@Override
public boolean equals(Object o) {
if (o == this) {
return true;
}
if (!(o instanceof GraphOperation)) {
return false;
}
GraphOperation that = (GraphOperation) o;
if (graph != that.graph) {
return false;
}
// The graph object is known to be identical here, so this one
// reference is sufficient to validate the use of native pointers
// in both objects.
Graph.Reference r = graph.ref();
try {
return getUnsafeNativeHandle() == that.getUnsafeNativeHandle();
} finally {
r.close();
}
}
@Override
public int inputListLength(final String name) {
Graph.Reference r = graph.ref();
try {
return inputListLength(getUnsafeNativeHandle(), name);
} finally {
r.close();
}
}
@Override
long getUnsafeNativeHandle(int outputIdx) {
return getUnsafeNativeHandle();
}
@Override
long[] shape(int outputIdx) {
Graph.Reference r = graph.ref();
try {
return shape(r.nativeHandle(), getUnsafeNativeHandle(), outputIdx);
} finally {
r.close();
}
}
@Override
DataType dtype(int outputIdx) {
Graph.Reference r = graph.ref();
try {
return DataType.fromC(dtype(r.nativeHandle(), getUnsafeNativeHandle(), outputIdx));
} finally {
r.close();
}
}
@Override
Tensor<?> tensor(int outputIdx) {
throw new IllegalStateException("Graph tensors must be fetched by running a session");
}
long getUnsafeNativeHandle() {
return unsafeNativeHandle;
}
private final Graph graph;
private final long unsafeNativeHandle;
private static native String name(long handle);
private static native String type(long handle);
private static native int numOutputs(long handle);
private static native int outputListLength(long handle, String name);
private static native int inputListLength(long handle, String name);
private static native long[] shape(long graphHandle, long opHandle, int output);
private static native int dtype(long graphHandle, long opHandle, int output);
}
tensorflow/java/src/main/java/org/tensorflow/GraphOperationBuilder.java
+344
View File
@@ -0,0 +1,344 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import java.nio.charset.Charset;
/** An {@link OperationBuilder} for adding {@link GraphOperation}s to a {@link Graph}. */
public final class GraphOperationBuilder implements OperationBuilder {
GraphOperationBuilder(Graph graph, String type, String name) {
this.graph = graph;
Graph.Reference r = graph.ref();
try {
this.unsafeNativeHandle = allocate(r.nativeHandle(), type, name);
} finally {
r.close();
}
}
/**
* Add the {@link GraphOperation} being built to the {@link Graph}.
*
* <p>The OperationBuilder is not usable after build() returns.
*/
@Override
public GraphOperation build() {
Graph.Reference r = graph.ref();
try {
GraphOperation op = new GraphOperation(graph, finish(unsafeNativeHandle));
unsafeNativeHandle = 0;
return op;
} finally {
r.close();
}
}
@Override
public GraphOperationBuilder addControlInput(Operation control) {
if (!(control instanceof GraphOperation)) {
throw new IllegalArgumentException(
"Only GraphOperation instances can be used as control inputs");
}
Graph.Reference r = graph.ref();
try {
addControlInput(unsafeNativeHandle, ((GraphOperation) control).getUnsafeNativeHandle());
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder addInput(Output<?> input) {
Graph.Reference r = graph.ref();
try {
addInput(unsafeNativeHandle, input.getUnsafeNativeHandle(), input.index());
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder addInputList(Output<?>[] inputs) {
Graph.Reference r = graph.ref();
try {
long[] opHandles = new long[inputs.length];
int[] indices = new int[inputs.length];
for (int i = 0; i < inputs.length; ++i) {
opHandles[i] = inputs[i].getUnsafeNativeHandle();
indices[i] = inputs[i].index();
}
addInputList(unsafeNativeHandle, opHandles, indices);
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder setDevice(String device) {
Graph.Reference r = graph.ref();
try {
setDevice(unsafeNativeHandle, device);
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder setAttr(String name, String value) {
setAttr(name, value.getBytes(Charset.forName("UTF-8")));
return this;
}
@Override
public GraphOperationBuilder setAttr(String name, byte[] value) {
Graph.Reference r = graph.ref();
try {
setAttrString(unsafeNativeHandle, name, value);
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder setAttr(String name, long value) {
Graph.Reference r = graph.ref();
try {
setAttrInt(unsafeNativeHandle, name, value);
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder setAttr(String name, long[] value) {
Graph.Reference r = graph.ref();
try {
setAttrIntList(unsafeNativeHandle, name, value);
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder setAttr(String name, float value) {
Graph.Reference r = graph.ref();
try {
setAttrFloat(unsafeNativeHandle, name, value);
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder setAttr(String name, float[] value) {
Graph.Reference r = graph.ref();
try {
setAttrFloatList(unsafeNativeHandle, name, value);
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder setAttr(String name, boolean value) {
Graph.Reference r = graph.ref();
try {
setAttrBool(unsafeNativeHandle, name, value);
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder setAttr(String name, boolean[] value) {
Graph.Reference r = graph.ref();
try {
setAttrBoolList(unsafeNativeHandle, name, value);
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder setAttr(String name, DataType value) {
Graph.Reference r = graph.ref();
try {
setAttrType(unsafeNativeHandle, name, value.c());
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder setAttr(String name, DataType[] value) {
int[] ctypes = new int[value.length];
for (int i = 0; i < value.length; ++i) {
ctypes[i] = value[i].c();
}
Graph.Reference r = graph.ref();
try {
setAttrTypeList(unsafeNativeHandle, name, ctypes);
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder setAttr(String name, Tensor<?> value) {
Graph.Reference r = graph.ref();
try {
setAttrTensor(unsafeNativeHandle, name, value.getNativeHandle());
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder setAttr(String name, Tensor<?>[] value) {
long[] handles = new long[value.length];
int idx = 0;
for (Tensor<?> t : value) {
handles[idx++] = t.getNativeHandle();
}
Graph.Reference r = graph.ref();
try {
setAttrTensorList(unsafeNativeHandle, name, handles);
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder setAttr(String name, Shape value) {
Graph.Reference r = graph.ref();
try {
setAttrShape(unsafeNativeHandle, name, value.asArray(), value.numDimensions());
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder setAttr(String name, Shape[] value) {
int[] numDimensions = new int[value.length];
int totalNumDimensions = 0;
for (int idx = 0; idx < value.length; ++idx) {
int n = value[idx].numDimensions();
numDimensions[idx] = n;
if (n > 0) {
totalNumDimensions += n;
}
}
// Flatten the shapes into a single array to avoid too much overhead in the
// native part
long[] shapes = new long[totalNumDimensions];
int shapeIdx = 0;
for (Shape shape : value) {
if (shape.numDimensions() > 0) {
for (long dim : shape.asArray()) {
shapes[shapeIdx++] = dim;
}
}
}
Graph.Reference r = graph.ref();
try {
setAttrShapeList(unsafeNativeHandle, name, shapes, numDimensions);
} finally {
r.close();
}
return this;
}
@Override
public GraphOperationBuilder setAttr(String name, String[] value) {
Charset utf8 = Charset.forName("UTF-8");
Object[] objects = new Object[value.length];
for (int i = 0; i < value.length; ++i) {
objects[i] = value[i].getBytes(utf8);
}
Graph.Reference r = graph.ref();
try {
setAttrStringList(unsafeNativeHandle, name, objects);
} finally {
r.close();
}
return this;
}
private long unsafeNativeHandle;
private Graph graph;
private static native long allocate(long graphHandle, String type, String name);
private static native long finish(long handle);
private static native void addInput(long handle, long opHandle, int index);
private static native void addInputList(long handle, long[] opHandles, int[] indices);
private static native void addControlInput(long handle, long opHandle);
private static native void setDevice(long handle, String device);
// The names of all the setAttr* family functions below correspond to the C library types, not the
// Java library types. Roughly, setAttrFoo calls the TensorFlow C library function: TF_SetAttrFoo.
private static native void setAttrString(long handle, String name, byte[] value);
private static native void setAttrInt(long handle, String name, long value);
private static native void setAttrIntList(long handle, String name, long[] value);
private static native void setAttrFloat(long handle, String name, float value);
private static native void setAttrFloatList(long handle, String name, float[] value);
private static native void setAttrBool(long handle, String name, boolean value);
private static native void setAttrBoolList(long handle, String name, boolean[] value);
private static native void setAttrType(long handle, String name, int type);
private static native void setAttrTypeList(long handle, String name, int[] type);
private static native void setAttrTensor(long handle, String name, long tensorHandle);
private static native void setAttrTensorList(long handle, String name, long[] tensorHandle);
private static native void setAttrShape(long handle, String name, long[] shape, int numDims);
private static native void setAttrShapeList(
long handle, String name, long[] shapes, int[] numDims);
private static native void setAttrStringList(long handle, String name, Object[] value);
}
tensorflow/java/src/main/java/org/tensorflow/NativeLibrary.java
+275
View File
@@ -0,0 +1,275 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
/**
* Helper class for loading the TensorFlow Java native library.
*
* <p>The Java TensorFlow bindings require a native (JNI) library. This library
* (libtensorflow_jni.so on Linux, libtensorflow_jni.dylib on OS X, tensorflow_jni.dll on Windows)
* can be made available to the JVM using the java.library.path System property (e.g., using
* -Djava.library.path command-line argument). However, doing so requires an additional step of
* configuration.
*
* <p>Alternatively, the native libraries can be packaed in a .jar, making them easily usable from
* build systems like Maven. However, in such cases, the native library has to be extracted from the
* .jar archive.
*
* <p>NativeLibrary.load() takes care of this. First looking for the library in java.library.path
* and failing that, it tries to find the OS and architecture specific version of the library in the
* set of ClassLoader resources (under org/tensorflow/native/OS-ARCH). The resources paths used for
* lookup must be consistent with any packaging (such as on Maven Central) of the TensorFlow Java
* native libraries.
*/
final class NativeLibrary {
private static final boolean DEBUG =
System.getProperty("org.tensorflow.NativeLibrary.DEBUG") != null;
private static final String JNI_LIBNAME = "tensorflow_jni";
public static void load() {
if (isLoaded() || tryLoadLibrary()) {
// Either:
// (1) The native library has already been statically loaded, OR
// (2) The required native code has been statically linked (through a custom launcher), OR
// (3) The native code is part of another library (such as an application-level library)
// that has already been loaded. For example, tensorflow/tools/android/test and
// tensorflow/tools/android/inference_interface include the required native code in
// differently named libraries.
//
// Doesn't matter how, but it seems the native code is loaded, so nothing else to do.
return;
}
// Native code is not present, perhaps it has been packaged into the .jar file containing this.
// Extract the JNI library itself
final String jniLibName = System.mapLibraryName(JNI_LIBNAME);
final String jniResourceName = makeResourceName(jniLibName);
log("jniResourceName: " + jniResourceName);
final InputStream jniResource =
NativeLibrary.class.getClassLoader().getResourceAsStream(jniResourceName);
// Extract the JNI's dependency
final String frameworkLibName =
getVersionedLibraryName(System.mapLibraryName("tensorflow_framework"));
final String frameworkResourceName = makeResourceName(frameworkLibName);
log("frameworkResourceName: " + frameworkResourceName);
final InputStream frameworkResource =
NativeLibrary.class.getClassLoader().getResourceAsStream(frameworkResourceName);
// Do not complain if the framework resource wasn't found. This may just mean that we're
// building with --config=monolithic (in which case it's not needed and not included).
if (jniResource == null) {
throw new UnsatisfiedLinkError(
String.format(
"Cannot find TensorFlow native library for OS: %s, architecture: %s. See "
+ "https://github.com/tensorflow/tensorflow/tree/master/tensorflow/java/README.md"
+ " for possible solutions (such as building the library from source). Additional"
+ " information on attempts to find the native library can be obtained by adding"
+ " org.tensorflow.NativeLibrary.DEBUG=1 to the system properties of the JVM.",
os(), architecture()));
}
try {
// Create a temporary directory for the extracted resource and its dependencies.
final File tempPath = createTemporaryDirectory();
// Deletions are in the reverse order of requests, so we need to request that the directory be
// deleted first, so that it is empty when the request is fulfilled.
tempPath.deleteOnExit();
final String tempDirectory = tempPath.getCanonicalPath();
if (frameworkResource != null) {
extractResource(frameworkResource, frameworkLibName, tempDirectory);
} else {
log(
frameworkResourceName
+ " not found. This is fine assuming "
+ jniResourceName
+ " is not built to depend on it.");
}
System.load(extractResource(jniResource, jniLibName, tempDirectory));
} catch (IOException e) {
throw new UnsatisfiedLinkError(
String.format(
"Unable to extract native library into a temporary file (%s)", e.toString()));
}
}
private static boolean tryLoadLibrary() {
try {
System.loadLibrary(JNI_LIBNAME);
return true;
} catch (UnsatisfiedLinkError e) {
log("tryLoadLibraryFailed: " + e.getMessage());
return false;
}
}
private static boolean isLoaded() {
try {
TensorFlow.version();
log("isLoaded: true");
return true;
} catch (UnsatisfiedLinkError e) {
return false;
}
}
private static boolean resourceExists(String baseName) {
return NativeLibrary.class.getClassLoader().getResource(makeResourceName(baseName)) != null;
}
private static String getVersionedLibraryName(String libFilename) {
final String versionName = getMajorVersionNumber();
// If we're on darwin, the versioned libraries look like blah.1.dylib.
final String darwinSuffix = ".dylib";
if (libFilename.endsWith(darwinSuffix)) {
final String prefix = libFilename.substring(0, libFilename.length() - darwinSuffix.length());
if (versionName != null) {
final String darwinVersionedLibrary = prefix + "." + versionName + darwinSuffix;
if (resourceExists(darwinVersionedLibrary)) {
return darwinVersionedLibrary;
}
} else {
// If we're here, we're on darwin, but we couldn't figure out the major version number. We
// already tried the library name without any changes, but let's do one final try for the
// library with a .so suffix.
final String darwinSoName = prefix + ".so";
if (resourceExists(darwinSoName)) {
return darwinSoName;
}
}
} else if (libFilename.endsWith(".so")) {
// Libraries ending in ".so" are versioned like "libfoo.so.1", so try that.
final String versionedSoName = libFilename + "." + versionName;
if (versionName != null && resourceExists(versionedSoName)) {
return versionedSoName;
}
}
// Otherwise, we've got no idea.
return libFilename;
}
/**
* Returns the major version number of this TensorFlow Java API, or {@code null} if it cannot be
* determined.
*/
private static String getMajorVersionNumber() {
InputStream resourceStream =
NativeLibrary.class.getClassLoader().getResourceAsStream("tensorflow-version-info");
if (resourceStream == null) {
return null;
}
try {
Properties props = new Properties();
props.load(resourceStream);
String version = props.getProperty("version");
// expecting a string like 1.14.0, we want to get the first '1'.
int dotIndex;
if (version == null || (dotIndex = version.indexOf('.')) == -1) {
return null;
}
String majorVersion = version.substring(0, dotIndex);
try {
Integer.parseInt(majorVersion);
return majorVersion;
} catch (NumberFormatException unused) {
return null;
}
} catch (IOException e) {
log("failed to load tensorflow version info.");
return null;
}
}
private static String extractResource(
InputStream resource, String resourceName, String extractToDirectory) throws IOException {
final File dst = new File(extractToDirectory, resourceName);
dst.deleteOnExit();
final String dstPath = dst.toString();
log("extracting native library to: " + dstPath);
final long nbytes = copy(resource, dst);
log(String.format("copied %d bytes to %s", nbytes, dstPath));
return dstPath;
}
private static String os() {
final String p = System.getProperty("os.name").toLowerCase();
if (p.contains("linux")) {
return "linux";
} else if (p.contains("os x") || p.contains("darwin")) {
return "darwin";
} else if (p.contains("windows")) {
return "windows";
} else {
return p.replaceAll("\\s", "");
}
}
private static String architecture() {
final String arch = System.getProperty("os.arch").toLowerCase();
return (arch.equals("amd64")) ? "x86_64" : arch;
}
private static void log(String msg) {
if (DEBUG) {
System.err.println("org.tensorflow.NativeLibrary: " + msg);
}
}
private static String makeResourceName(String baseName) {
return "org/tensorflow/native/" + String.format("%s-%s/", os(), architecture()) + baseName;
}
private static long copy(InputStream src, File dstFile) throws IOException {
FileOutputStream dst = new FileOutputStream(dstFile);
try {
byte[] buffer = new byte[1 << 20]; // 1MB
long ret = 0;
int n = 0;
while ((n = src.read(buffer)) >= 0) {
dst.write(buffer, 0, n);
ret += n;
}
return ret;
} finally {
dst.close();
src.close();
}
}
// Shamelessly adapted from Guava to avoid using java.nio, for Android API
// compatibility.
private static File createTemporaryDirectory() {
File baseDirectory = new File(System.getProperty("java.io.tmpdir"));
String directoryName = "tensorflow_native_libraries-" + System.currentTimeMillis() + "-";
for (int attempt = 0; attempt < 1000; attempt++) {
File temporaryDirectory = new File(baseDirectory, directoryName + attempt);
if (temporaryDirectory.mkdir()) {
return temporaryDirectory;
}
}
throw new IllegalStateException(
"Could not create a temporary directory (tried to make "
+ directoryName
+ "*) to extract TensorFlow native libraries.");
}
private NativeLibrary() {}
}
tensorflow/java/src/main/java/org/tensorflow/Operand.java
+48
View File
@@ -0,0 +1,48 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
/**
* Interface implemented by operands of a TensorFlow operation.
*
* <p>Example usage:
*
* <pre>{@code
* // The "decodeJpeg" operation can be used as an operand to the "cast" operation
* Operand<UInt8> decodeJpeg = ops.image().decodeJpeg(...);
* ops.math().cast(decodeJpeg, DataType.FLOAT);
*
* // The output "y" of the "unique" operation can be used as an operand to the "cast" operation
* Output<Integer> y = ops.array().unique(...).y();
* ops.math().cast(y, Float.class);
*
* // The "split" operation can be used as operand list to the "concat" operation
* Iterable<? extends Operand<Float>> split = ops.array().split(...);
* ops.array().concat(0, split);
* }</pre>
*/
public interface Operand<T> {
/**
* Returns the symbolic handle of a tensor.
*
* <p>Inputs to TensorFlow operations are outputs of another TensorFlow operation. This method is
* used to obtain a symbolic handle that represents the computation of the input.
*
* @see OperationBuilder#addInput(Output)
*/
Output<T> asOutput();
}
tensorflow/java/src/main/java/org/tensorflow/Operation.java
+86
View File
@@ -0,0 +1,86 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
/**
* Performs computation on Tensors.
*
* <p>An Operation takes zero or more {@link Tensor}s (produced by other Operations) as input, and
* produces zero or more {@link Tensor}s as output.
*/
public interface Operation {
/** Returns the full name of the Operation. */
String name();
/**
* Returns the type of the operation, i.e., the name of the computation performed by the
* operation.
*/
String type();
/** Returns the number of tensors produced by this operation. */
int numOutputs();
/**
* Returns the size of the list of Tensors produced by this operation.
*
* <p>An Operation has multiple named outputs, each of which produces either a single tensor or a
* list of tensors. This method returns the size of the list of tensors for a specific named
* output of the operation.
*
* @param name identifier of the list of tensors (of which there may be many) produced by this
* operation.
* @return the size of the list of Tensors produced by this named output.
* @throws IllegalArgumentException if this operation has no output with the provided name.
*/
int outputListLength(final String name);
/**
* Returns symbolic handles to a list of tensors produced by this operation.
*
* @param idx index of the first tensor of the list
* @param length number of tensors in the list
* @return array of {@code Output}
*/
Output<?>[] outputList(int idx, int length);
/**
* Returns a symbolic handle to one of the tensors produced by this operation.
*
* <p>Warning: Does not check that the type of the tensor matches T. It is recommended to call
* this method with an explicit type parameter rather than letting it be inferred, e.g. {@code
* operation.<Integer>output(0)}
*
* @param <T> The expected element type of the tensors produced by this output.
* @param idx The index of the output among the outputs produced by this operation.
*/
<T> Output<T> output(int idx);
/**
* Returns the size of the given inputs list of Tensors for this operation.
*
* <p>An Operation has multiple named inputs, each of which contains either a single tensor or a
* list of tensors. This method returns the size of the list of tensors for a specific named input
* of the operation.
*
* @param name identifier of the list of tensors (of which there may be many) inputs to this
* operation.
* @return the size of the list of Tensors produced by this named input.
* @throws IllegalArgumentException if this operation has no input with the provided name.
*/
int inputListLength(final String name);
}
tensorflow/java/src/main/java/org/tensorflow/OperationBuilder.java
+224
View File
@@ -0,0 +1,224 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
/**
* A builder for {@link Operation}s.
*
* <p>For example, the following uses the builder to create an operation that produces the constant
* "3" as its output:
*
* <pre>{@code
* // env is an ExecutionEnvironment, such as a Graph instance.
* try (Tensor c1 = Tensor.create(3.0f)) {
* env.opBuilder("Const", "MyConst")
* .setAttr("dtype", c1.dataType())
* .setAttr("value", c1)
* .build();
* }
* }</pre>
*/
public interface OperationBuilder {
/**
* Build the {@link Operation}.
*
* <p>The following action will also be performed depending on the current execution environment.
*
* <ul>
* <li>In eager mode, the result of the operation will be computed immediately.
* <li>In graph mode, the operation will be added as a node to the graph to be executed later,
* when running a {@link Session}.
* </ul>
*
* <p>The OperationBuilder is not usable after build() returns.
*/
public Operation build();
/**
* Add the output of another operation as the next input of the operation being built.
*
* @param input {@link Output} supposed to be the input of the operation being built.
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder addInput(Output<?> input);
/**
* Add the outputs of another operation as the next inputs of the operation being built.
*
* @param inputs list of {@link Output} supposed to be the inputs of the operation being built.
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder addInputList(Output<?>[] inputs);
/**
* Ensure that the operation does not execute before the control operation does.
*
* <p>A control input is an Operation that must be executed before running the operation currently
* being built.
*
* <p>For example, an Assert operation may be added as a control input for this operation. The
* Assert now behaves as a pre-condition that will always verify itself before running the
* operation.
*
* @param control operation that must be executed before running this operation.
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder addControlInput(Operation control);
/**
* Set the device requested for computing the operation being built.
*
* @param device the requested device, as a string
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setDevice(String device);
/**
* Set the string values of an attribute of the operation being built.
*
* @param name attribute name
* @param value attribute values
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setAttr(String name, String[] value);
/**
* Set the string value of an attribute of the operation being built.
*
* @param name attribute name
* @param value attribute value
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setAttr(String name, String value);
/**
* Set the byte values of an attribute of the operation being built.
*
* @param name attribute name
* @param value attribute values
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setAttr(String name, byte[] value);
/**
* Set the long value of an attribute of the operation being built.
*
* @param name attribute name
* @param value attribute value
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setAttr(String name, long value);
/**
* Set the long values of an attribute of the operation being built.
*
* @param name attribute name
* @param value attribute values
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setAttr(String name, long[] value);
/**
* Set the float value of an attribute of the operation being built.
*
* @param name attribute name
* @param value attribute value
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setAttr(String name, float value);
/**
* Set the float values of an attribute of the operation being built.
*
* @param name attribute name
* @param value attribute values
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setAttr(String name, float[] value);
/**
* Set the boolean value of an attribute of the operation being built.
*
* @param name attribute name
* @param value attribute value
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setAttr(String name, boolean value);
/**
* Set the boolean values of an attribute of the operation being built.
*
* @param name attribute name
* @param value attribute values
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setAttr(String name, boolean[] value);
/**
* Set the type value of an attribute of the operation being built.
*
* @param name attribute name
* @param value attribute value
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setAttr(String name, DataType value);
/**
* Set the type values of an attribute of the operation being built.
*
* @param name attribute name
* @param value attribute values
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setAttr(String name, DataType[] value);
/**
* Set the tensor value of an attribute of the operation being built.
*
* @param name attribute name
* @param value attribute value
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setAttr(String name, Tensor<?> value);
/**
* Set the tensor values of an attribute of the operation being built.
*
* @param name attribute name
* @param value attribute values
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setAttr(String name, Tensor<?>[] value);
/**
* Set the shape value of an attribute of the operation being built.
*
* @param name attribute name
* @param value attribute value
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setAttr(String name, Shape value);
/**
* Set the shape values of an attribute of the operation being built.
*
* @param name attribute name
* @param value attribute values
* @return the OperationBuilder instance for chaining.
*/
public OperationBuilder setAttr(String name, Shape[] value);
}
tensorflow/java/src/main/java/org/tensorflow/Output.java
+108
View File
@@ -0,0 +1,108 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import java.util.Objects;
/**
* A symbolic handle to a tensor produced by an {@link Operation}.
*
* <p>An {@code Output<T>} is a symbolic handle to a {@code Tensor<T>}. The value of the tensor is
* computed by executing the {@link Operation} in a {@link Session}.
*
* <p>By implementing the {@link Operand} interface, instances of this class also act as operands to
* {@link org.tensorflow.op.Op Op} instances.
*/
public final class Output<T> implements Operand<T> {
/** Returns the Operation that will produce the tensor referred to by this Output. */
public Operation op() {
return operation;
}
/** Returns the index into the outputs of the Operation. */
public int index() {
return index;
}
/** Returns the (possibly partially known) shape of the tensor referred to by this Output. */
public Shape shape() {
return new Shape(operation.shape(index));
}
/** Returns the DataType of the tensor referred to by this Output. */
public DataType dataType() {
return operation.dtype(index);
}
/**
* Returns the tensor at this output.
*
* <p>This operation is only supported on the outputs of an operation executed eagerly. For graph
* environments, output tensors must be fetched by running a session, using {@link
* Session.Runner#fetch(Output)}.
*
* @return tensor
* @throws IllegalStateException if this output results from a graph
* @see EagerSession
*/
@SuppressWarnings("unchecked")
public Tensor<T> tensor() {
return (Tensor<T>) operation.tensor(index);
}
@Override
public Output<T> asOutput() {
return this;
}
@Override
public int hashCode() {
return Objects.hash(operation, index);
}
@Override
public boolean equals(Object o) {
if (o == this) {
return true;
}
if (o instanceof Output<?>) {
Output<?> that = (Output<?>) o;
return index == that.index && operation.equals(that.operation);
}
return false;
}
@Override
public String toString() {
return String.format(
"<%s '%s:%d' shape=%s dtype=%s>",
operation.type(), operation.name(), index, shape().toString(), dataType());
}
/** Handle to the idx-th output of the Operation {@code op}. */
Output(AbstractOperation op, int idx) {
operation = op;
index = idx;
}
long getUnsafeNativeHandle() {
return operation.getUnsafeNativeHandle(index);
}
private final AbstractOperation operation;
private final int index;
}
tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java
+172
View File
@@ -0,0 +1,172 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
/**
* SavedModelBundle represents a model loaded from storage.
*
* <p>The model consists of a description of the computation (a {@link Graph}), a {@link Session}
* with tensors (e.g., parameters or variables in the graph) initialized to values saved in storage,
* and a description of the model (a serialized representation of a <a
* href="https://www.tensorflow.org/code/tensorflow/core/protobuf/meta_graph.proto">MetaGraphDef
* protocol buffer</a>).
*/
public class SavedModelBundle implements AutoCloseable {
/** Options for loading a SavedModel. */
public static final class Loader {
/** Load a <code>SavedModelBundle</code> with the configured options. */
public SavedModelBundle load() {
return SavedModelBundle.load(exportDir, tags, configProto, runOptions);
}
/**
* Sets options to use when executing model initialization operations.
*
* @param options Serialized <a
* href="https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto">RunOptions
* protocol buffer</a>.
*/
public Loader withRunOptions(byte[] options) {
this.runOptions = options;
return this;
}
/**
* Set configuration of the <code>Session</code> object created when loading the model.
*
* @param configProto Serialized <a
* href="https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto">ConfigProto
* protocol buffer</a>.
*/
public Loader withConfigProto(byte[] configProto) {
this.configProto = configProto;
return this;
}
/**
* Sets the set of tags that identify the specific graph in the saved model to load.
*
* @param tags the tags identifying the specific MetaGraphDef to load.
*/
public Loader withTags(String... tags) {
this.tags = tags;
return this;
}
private Loader(String exportDir) {
this.exportDir = exportDir;
}
private String exportDir = null;
private String[] tags = null;
private byte[] configProto = null;
private byte[] runOptions = null;
}
/**
* Load a saved model from an export directory. The model that is being loaded should be created
* using the <a href="https://www.tensorflow.org/api_docs/python/tf/saved_model">Saved Model
* API</a>.
*
* <p>This method is a shorthand for:
*
* <pre>{@code
* SavedModelBundle.loader().withTags(tags).load();
* }</pre>
*
* @param exportDir the directory path containing a saved model.
* @param tags the tags identifying the specific metagraphdef to load.
* @return a bundle containing the graph and associated session.
*/
public static SavedModelBundle load(String exportDir, String... tags) {
return loader(exportDir).withTags(tags).load();
}
/**
* Load a saved model.
*
* <p/>Returns a <code>Loader</code> object that can set configuration options before actually
* loading the model,
*
* @param exportDir the directory path containing a saved model.
*/
public static Loader loader(String exportDir) {
return new Loader(exportDir);
}
/**
* Returns the serialized <a
* href="https://www.tensorflow.org/code/tensorflow/core/protobuf/meta_graph.proto">MetaGraphDef
* protocol buffer</a> associated with the saved model.
*/
public byte[] metaGraphDef() {
return metaGraphDef;
}
/** Returns the graph that describes the computation performed by the model. */
public Graph graph() {
return graph;
}
/**
* Returns the {@link Session} with which to perform computation using the model.
*
* @return the initialized session
*/
public Session session() {
return session;
}
/**
* Releases resources (the {@link Graph} and {@link Session}) associated with the saved model
* bundle.
*/
@Override
public void close() {
session.close();
graph.close();
}
private final Graph graph;
private final Session session;
private final byte[] metaGraphDef;
private SavedModelBundle(Graph graph, Session session, byte[] metaGraphDef) {
this.graph = graph;
this.session = session;
this.metaGraphDef = metaGraphDef;
}
/**
* Create a SavedModelBundle object from a handle to the C TF_Graph object and to the C TF_Session
* object, plus the serialized MetaGraphDef.
*
* <p>Invoked from the native load method. Takes ownership of the handles.
*/
private static SavedModelBundle fromHandle(
long graphHandle, long sessionHandle, byte[] metaGraphDef) {
Graph graph = new Graph(graphHandle);
Session session = new Session(graph, sessionHandle);
return new SavedModelBundle(graph, session, metaGraphDef);
}
private static native SavedModelBundle load(
String exportDir, String[] tags, byte[] config, byte[] runOptions);
static {
TensorFlow.init();
}
}
tensorflow/java/src/main/java/org/tensorflow/Server.java
+133
View File
@@ -0,0 +1,133 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
/**
* An in-process TensorFlow server, for use in distributed training.
*
* <p>A {@code Server} instance encapsulates a set of devices and a {@link org.tensorflow.Session}
* target that can participate in distributed training. A server belongs to a cluster (specified by
* a {@code ClusterSpec}), and corresponds to a particular task in a named job. The server can
* communicate with any other server in the same cluster. The server will not serve any requests
* until {@link #start()} is invoked. The server will stop serving requests once {@link #stop()} or
* {@link #close()} is invoked. Be aware that {@link #close()} method stops the server if it is
* running.
*
* <p><b>WARNING:</b> A {@code Server} owns resources that <b>must</b> be explicitly freed by
* invoking {@link #close()}.
*
* <p>Instances of a {@code Server} are thread-safe.
*
* <p>Using example:
*
* <pre>{@code
* import org.tensorflow.Server;
* import org.tensorflow.distruntime.ClusterDef;
* import org.tensorflow.distruntime.JobDef;
* import org.tensorflow.distruntime.ServerDef;
*
* ClusterDef clusterDef = ClusterDef.newBuilder()
* .addJob(JobDef.newBuilder()
* .setName("worker")
* .putTasks(0, "localhost:4321")
* .build()
* ).build();
*
* ServerDef serverDef = ServerDef.newBuilder()
* .setCluster(clusterDef)
* .setJobName("worker")
* .setTaskIndex(0)
* .setProtocol("grpc")
* .build();
*
* try (Server srv = new Server(serverDef.toByteArray())) {
* srv.start();
* srv.join();
* }
* }</pre>
*/
public final class Server implements AutoCloseable {
/**
* Constructs a new instance of server.
*
* @param serverDef Server definition specified as a serialized <a
* href="https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/tensorflow_server.proto">ServerDef</a>
* protocol buffer.
*/
public Server(byte[] serverDef) {
nativeHandle = allocate(serverDef);
}
/** Starts an in-process TensorFlow server. */
public synchronized void start() {
start(nativeHandle);
}
/** Stops an in-process TensorFlow server. */
public synchronized void stop() {
stop(nativeHandle);
}
/** Blocks until the server has been successfully stopped. */
public void join() {
long handle = 0;
synchronized (this) {
handle = nativeHandle;
if (handle != 0) {
numJoining++;
}
}
try {
join(handle);
} finally {
synchronized (this) {
if (handle != 0) {
numJoining--;
}
notifyAll();
}
}
}
/** Destroy an in-process TensorFlow server, frees memory. */
@Override
public synchronized void close() throws InterruptedException {
stop();
while (numJoining > 0) {
wait();
}
delete(nativeHandle);
nativeHandle = 0;
}
private static native long allocate(byte[] serverDef);
private static native void start(long nativeHandle);
private static native void stop(long nativeHandle);
private static native void join(long nativeHandle);
private static native void delete(long nativeHandle);
private long nativeHandle;
private int numJoining;
static {
TensorFlow.init();
}
}
tensorflow/java/src/main/java/org/tensorflow/Session.java
+491
View File
@@ -0,0 +1,491 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import java.util.ArrayList;
import java.util.List;
/**
* Driver for {@link Graph} execution.
*
* <p>A {@code Session} instance encapsulates the environment in which {@link Operation}s in a
* {@link Graph} are executed to compute {@link Tensor Tensors}. For example:
*
* <pre>{@code
* // Let's say graph is an instance of the Graph class
* // for the computation y = 3 * x
*
* try (Session s = new Session(graph)) {
* try (Tensor x = Tensor.create(2.0f);
* Tensor y = s.runner().feed("x", x).fetch("y").run().get(0)) {
* System.out.println(y.floatValue()); // Will print 6.0f
* }
* try (Tensor x = Tensor.create(1.1f);
* Tensor y = s.runner().feed("x", x).fetch("y").run().get(0)) {
* System.out.println(y.floatValue()); // Will print 3.3f
* }
* }
* }</pre>
*
* <p><b>WARNING:</b>A {@code Session} owns resources that <b>must</b> be explicitly freed by
* invoking {@link #close()}.
*
* <p>Instances of a Session are thread-safe.
*/
public final class Session implements AutoCloseable {
/** Construct a new session with the associated {@link Graph}. */
public Session(Graph g) {
this(g, null);
}
/**
* Construct a new session with the associated {@link Graph} and configuration options.
*
* @param g The {@link Graph} the created Session will operate on.
* @param config Configuration parameters for the session specified as a serialized <a
* href="https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto">ConfigProto</a>
* protocol buffer.
* @throws IllegalArgumentException if the config is not a valid serialization of the ConfigProto
* protocol buffer.
*/
public Session(Graph g, byte[] config) {
graph = g;
Graph.Reference r = g.ref();
try {
nativeHandle =
(config == null) ? allocate(r.nativeHandle()) : allocate2(r.nativeHandle(), null, config);
graphRef = g.ref();
} finally {
r.close();
}
}
/** Wrap an existing session with the associated {@link Graph}. */
Session(Graph g, long nativeHandle) {
graph = g;
this.nativeHandle = nativeHandle;
graphRef = g.ref();
}
/**
* Release resources associated with the Session.
*
* <p>Blocks until there are no active executions ({@link Session.Runner#run()} calls). A Session
* is not usable after close returns.
*/
@Override
public void close() {
graphRef.close();
synchronized (nativeHandleLock) {
if (nativeHandle == 0) {
return;
}
while (numActiveRuns > 0) {
try {
nativeHandleLock.wait();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
// Possible leak of the Session and Graph in this case?
return;
}
}
delete(nativeHandle);
nativeHandle = 0;
}
}
/**
* Run {@link Operation}s and evaluate {@link Tensor Tensors}.
*
* <p>A Runner runs the necessary graph fragments to execute every {@link Operation} required to
* evaluate the {@link Tensor Tensors} to fetch. The {@link #feed(String,int,Tensor)} call allows
* callers to override the value of {@link Tensor Tensors} in the graph by substituting the
* provided {@link Tensor Tensors} for the outputs of the operations provided to {@link
* #feed(String,int,Tensor)}.
*/
public final class Runner {
/**
* Avoid evaluating {@code operation} and substitute {@code t} for the value it produces.
*
* @param operation Is either the string name of the operation, in which case this method is a
* shorthand for {@code feed(operation, 0)}, or it is a string of the form
* <tt>operation_name:output_index</tt> , in which case this method acts like {@code
* feed(operation_name, output_index)}. These colon-separated names are commonly used in the
* {@code SignatureDef} protocol buffer messages that are included in {@link
* SavedModelBundle#metaGraphDef()}.
*/
public Runner feed(String operation, Tensor<?> t) {
return feed(parseOutput(operation), t);
}
/**
* Avoid evaluating the {@code index}-th output of {@code operation} by substituting {@code t}
* for the value it produces.
*
* <p>Operations in a {@link Graph} can have multiple outputs, {@code index} identifies which
* one {@code t} is being provided for.
*/
public Runner feed(String operation, int index, Tensor<?> t) {
Operation op = operationByName(operation);
if (op != null) {
inputs.add(op.output(index));
inputTensors.add(t);
}
return this;
}
/**
* Use {@code t} instead of the Tensor referred to by executing the operation referred to by
* {@code operand}.
*/
public Runner feed(Operand<?> operand, Tensor<?> t) {
inputs.add(operand.asOutput());
inputTensors.add(t);
return this;
}
/**
* Make {@link #run()} return the output of {@code operation}.
*
* @param operation Is either the string name of the operation, in which case this method is a
* shorthand for {@code fetch(operation, 0)}, or it is a string of the form
* <tt>operation_name:output_index</tt> , in which case this method acts like {@code
* fetch(operation_name, output_index)}. These colon-separated names are commonly used in
* the {@code SignatureDef} protocol buffer messages that are included in {@link
* SavedModelBundle#metaGraphDef()}.
*/
public Runner fetch(String operation) {
return fetch(parseOutput(operation));
}
/**
* Make {@link #run()} return the {@code index}-th output of {@code operation}.
*
* <p>Operations in a {@link Graph} can have multiple outputs, {@code index} identifies which
* one to return.
*/
public Runner fetch(String operation, int index) {
Operation op = operationByName(operation);
if (op != null) {
outputs.add(op.output(index));
}
return this;
}
/**
* Makes {@link #run()} return the Tensor referred to by {@code output}.
*/
public Runner fetch(Output<?> output) {
outputs.add(output);
return this;
}
/**
* Makes {@link #run()} return the Tensor referred to by the output of {@code operand}.
*/
public Runner fetch(Operand<?> operand) {
return fetch(operand.asOutput());
}
/**
* Make {@link #run()} execute {@code operation}, but not return any evaluated {@link Tensor
* Tensors}.
*/
public Runner addTarget(String operation) {
GraphOperation op = operationByName(operation);
if (op != null) {
targets.add(op);
}
return this;
}
/**
* Make {@link #run()} execute {@code operation}, but not return any evaluated {@link Tensor
* Tensors}.
*
* @throws IllegalArgumentException if the operation is not a {@link GraphOperation}
*/
public Runner addTarget(Operation operation) {
if (!(operation instanceof GraphOperation)) {
throw new IllegalArgumentException(
"Operation of type "
+ operation.getClass().getName()
+ " is not supported in graph sessions");
}
targets.add((GraphOperation) operation);
return this;
}
/**
* Make {@link #run} execute {@code operand}, but not return any evaluated {@link Tensor
* Tensors}.
*/
public Runner addTarget(Operand<?> operand) {
return addTarget(operand.asOutput().op());
}
/**
* (Experimental method): set options (typically for debugging) for this run.
*
* <p>The options are presented as a serialized <a
* href="https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto">RunOptions
* protocol buffer</a>.
*
* <p>The org.tensorflow package is free of any protocol buffer dependencies in order to remain
* friendly to resource constrained systems (where something like <a
* href="https://github.com/google/protobuf/tree/master/javanano#nano-version">nanoproto</a> may
* be more appropriate). A cost of that is this lack of type-safety in this API function. This
* choice is under review and this function may be replaced by more type-safe equivalents at any
* time.
*/
public Runner setOptions(byte[] options) {
this.runOptions = options;
return this;
}
/**
* Execute the graph fragments necessary to compute all requested fetches.
*
* <p><b>WARNING:</b> The caller assumes ownership of all returned {@link Tensor Tensors}, i.e.,
* the caller must call {@link Tensor#close} on all elements of the returned list to free up
* resources.
*
* <p>TODO(ashankar): Reconsider the return type here. Two things in particular: (a) Make it
* easier for the caller to cleanup (perhaps returning something like AutoCloseableList in
* SessionTest.java), and (b) Evaluate whether the return value should be a list, or maybe a
* {@code Map<Output, Tensor>}?
*
* <p>TODO(andrewmyers): It would also be good if whatever is returned here made it easier to
* extract output tensors in a type-safe way.
*/
public List<Tensor<?>> run() {
return runHelper(false).outputs;
}
/**
* Execute graph fragments to compute requested fetches and return metadata about the run.
*
* <p>This is exactly like {@link #run()}, but in addition to the requested Tensors, also
* returns metadata about the graph execution in the form of a serialized <a
* href="https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto">RunMetadata
* protocol buffer</a>.
*/
public Run runAndFetchMetadata() {
return runHelper(true);
}
private Run runHelper(boolean wantMetadata) {
long[] inputTensorHandles = new long[inputTensors.size()];
long[] inputOpHandles = new long[inputs.size()];
int[] inputOpIndices = new int[inputs.size()];
long[] outputOpHandles = new long[outputs.size()];
int[] outputOpIndices = new int[outputs.size()];
long[] targetOpHandles = new long[targets.size()];
long[] outputTensorHandles = new long[outputs.size()];
// It's okay to use Operation.getUnsafeNativeHandle() here since the safety depends on the
// validity of the Graph and graphRef ensures that.
int idx = 0;
for (Tensor<?> t : inputTensors) {
inputTensorHandles[idx++] = t.getNativeHandle();
}
idx = 0;
for (Output<?> o : inputs) {
inputOpHandles[idx] = o.getUnsafeNativeHandle();
inputOpIndices[idx] = o.index();
idx++;
}
idx = 0;
for (Output<?> o : outputs) {
outputOpHandles[idx] = o.getUnsafeNativeHandle();
outputOpIndices[idx] = o.index();
idx++;
}
idx = 0;
for (GraphOperation op : targets) {
targetOpHandles[idx++] = op.getUnsafeNativeHandle();
}
Reference runRef = new Reference();
byte[] metadata = null;
try {
metadata =
Session.run(
nativeHandle,
runOptions,
inputTensorHandles,
inputOpHandles,
inputOpIndices,
outputOpHandles,
outputOpIndices,
targetOpHandles,
wantMetadata,
outputTensorHandles);
} finally {
runRef.close();
}
List<Tensor<?>> outputs = new ArrayList<Tensor<?>>();
for (long h : outputTensorHandles) {
try {
outputs.add(Tensor.fromHandle(h));
} catch (Exception e) {
for (Tensor<?> t : outputs) {
t.close();
}
outputs.clear();
throw e;
}
}
Run ret = new Run();
ret.outputs = outputs;
ret.metadata = metadata;
return ret;
}
private class Reference implements AutoCloseable {
public Reference() {
synchronized (nativeHandleLock) {
if (nativeHandle == 0) {
throw new IllegalStateException("run() cannot be called on the Session after close()");
}
++numActiveRuns;
}
}
@Override
public void close() {
synchronized (nativeHandleLock) {
if (nativeHandle == 0) {
return;
}
if (--numActiveRuns == 0) {
nativeHandleLock.notifyAll();
}
}
}
}
private GraphOperation operationByName(String opName) {
GraphOperation op = graph.operation(opName);
if (op == null) {
throw new IllegalArgumentException("No Operation named [" + opName + "] in the Graph");
}
return op;
}
@SuppressWarnings("rawtypes")
private Output<?> parseOutput(String opName) {
int colon = opName.lastIndexOf(':');
if (colon == -1 || colon == opName.length() - 1) {
return new Output(operationByName(opName), 0);
}
try {
String op = opName.substring(0, colon);
int index = Integer.parseInt(opName.substring(colon + 1));
return new Output(operationByName(op), index);
} catch (NumberFormatException e) {
return new Output(operationByName(opName), 0);
}
}
private ArrayList<Output<?>> inputs = new ArrayList<Output<?>>();
private ArrayList<Tensor<?>> inputTensors = new ArrayList<Tensor<?>>();
private ArrayList<Output<?>> outputs = new ArrayList<Output<?>>();
private ArrayList<GraphOperation> targets = new ArrayList<GraphOperation>();
private byte[] runOptions = null;
}
/** Create a Runner to execute graph operations and evaluate Tensors. */
public Runner runner() {
return new Runner();
}
/**
* Output tensors and metadata obtained when executing a session.
*
* <p>See {@link Runner#runAndFetchMetadata()}
*/
public static final class Run {
/** Tensors from requested fetches. */
public List<Tensor<?>> outputs;
/**
* (Experimental): Metadata about the run.
*
* <p>A serialized <a
* href="https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto">RunMetadata
* protocol buffer</a>. The org.tensorflow package is free of any protocol buffer dependencies
* in order to remain friendly to resource constrained systems (where something like <a
* href="https://github.com/google/protobuf/tree/master/javanano#nano-version">nanoproto</a> may
* be more appropriate). A cost of that is this opaque blob. This choice is under review and
* this field may be replaced by more type-safe equivalents at any time.
*/
public byte[] metadata;
}
private final Graph graph;
private final Graph.Reference graphRef;
private final Object nativeHandleLock = new Object();
private long nativeHandle;
private int numActiveRuns;
// TODO(ashankar): Remove after TensorFlow 1.2 has been released with allocate2().
private static native long allocate(long graphHandle);
private static native long allocate2(long graphHandle, String target, byte[] config);
private static native void delete(long handle);
/**
* Execute a session.
*
* <p>The author apologizes for the ugliness of the long argument list of this method. However,
* take solace in the fact that this is a private method meant to cross the JNI boundary.
*
* @param handle to the C API TF_Session object (Session.nativeHandle)
* @param runOptions serialized representation of a RunOptions protocol buffer, or null
* @param inputOpHandles (see inputOpIndices)
* @param inputOpIndices (see inputTensorHandles)
* @param inputTensorHandles together with inputOpHandles and inputOpIndices specifies the values
* that are being "fed" (do not need to be computed) during graph execution.
* inputTensorHandles[i] (which corresponds to a Tensor.nativeHandle) is considered to be the
* inputOpIndices[i]-th output of the Operation inputOpHandles[i]. Thus, it is required that
* inputOpHandles.length == inputOpIndices.length == inputTensorHandles.length.
* @param outputOpHandles (see outputOpIndices)
* @param outputOpIndices together with outputOpHandles identifies the set of values that should
* be computed. The outputOpIndices[i]-th output of the Operation outputOpHandles[i], It is
* required that outputOpHandles.length == outputOpIndices.length.
* @param targetOpHandles is the set of Operations in the graph that are to be executed but whose
* output will not be returned
* @param wantRunMetadata indicates whether metadata about this execution should be returned.
* @param outputTensorHandles will be filled in with handles to the outputs requested. It is
* required that outputTensorHandles.length == outputOpHandles.length.
* @return if wantRunMetadata is true, serialized representation of the RunMetadata protocol
* buffer, false otherwise.
*/
private static native byte[] run(
long handle,
byte[] runOptions,
long[] inputTensorHandles,
long[] inputOpHandles,
int[] inputOpIndices,
long[] outputOpHandles,
int[] outputOpIndices,
long[] targetOpHandles,
boolean wantRunMetadata,
long[] outputTensorHandles);
}
tensorflow/java/src/main/java/org/tensorflow/Shape.java
+133
View File
@@ -0,0 +1,133 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import java.util.Arrays;
/** The possibly partially known shape of a tensor produced by an operation. */
public final class Shape {
/** Create a Shape representing an unknown number of dimensions. */
public static Shape unknown() {
return new Shape(null);
}
/** Create a Shape representing a scalar value. */
public static Shape scalar() {
return new Shape(new long[0]);
}
/**
* Create a Shape representing an N-dimensional value.
*
* <p>Creates a Shape representing an N-dimensional value (N being at least 1), with the provided
* size for each dimension. A -1 indicates that the size of the corresponding dimension is
* unknown. For example:
*
* <pre>{@code
* // A 2-element vector.
* Shape vector = Shape.create(2);
*
* // A 2x3 matrix.
* Shape matrix = Shape.create(2, 3);
*
* // A matrix with 4 columns but an unknown number of rows.
* // This is typically used to indicate the shape of tensors that represent
* // a variable-sized batch of values. The Shape below might represent a
* // variable-sized batch of 4-element vectors.
* Shape batch = Shape.create(-1, 4);
* }</pre>
*/
public static Shape make(long firstDimensionSize, long... otherDimensionSizes) {
long[] shape = new long[otherDimensionSizes.length + 1];
shape[0] = firstDimensionSize;
System.arraycopy(otherDimensionSizes, 0, shape, 1, otherDimensionSizes.length);
return new Shape(shape);
}
/**
* Number of dimensions represented by this shape.
*
* @return -1 if the number of dimensions is unknown, 0 if the shape represents a scalar, 1 for a
* vector, 2 for a matrix etc.
*/
public int numDimensions() {
return shape == null ? -1 : shape.length;
}
/**
* The size of the i-th dimension.
*
* @return The size of the requested dimension or -1 if it is unknown.
*/
public long size(int i) {
return shape[i];
}
@Override
public int hashCode() {
return Arrays.hashCode(shape);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj instanceof Shape && Arrays.equals(this.shape, ((Shape) obj).shape)) {
return !hasUnknownDimension();
}
return super.equals(obj);
}
/** Succinct description of the shape meant for debugging. */
@Override
public String toString() {
if (shape == null) {
return "<unknown>";
}
return Arrays.toString(shape).replace("-1", "?");
}
// Package-private constructor.
Shape(long[] shape) {
this.shape = shape;
}
// Package-private accessor.
// The idea is that the public API does not expose the internal array.
long[] asArray() {
return shape;
}
private long[] shape;
private boolean hasUnknownDimension() {
if (shape == null) {
return true;
}
for (long dimension : shape) {
if (dimension == -1) {
return true;
}
}
return false;
}
}
tensorflow/java/src/main/java/org/tensorflow/Tensor.java
+840
View File
@@ -0,0 +1,840 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import java.lang.reflect.Array;
import java.nio.Buffer;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.DoubleBuffer;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.nio.LongBuffer;
import java.util.Arrays;
import java.util.HashMap;
/**
* A statically typed multi-dimensional array whose elements are of a type described by T.
*
* <p>Instances of a Tensor are <b>not</b> thread-safe.
*
* <p><b>WARNING:</b> Resources consumed by the Tensor object <b>must</b> be explicitly freed by
* invoking the {@link #close()} method when the object is no longer needed. For example, using a
* try-with-resources block:
*
* <pre>{@code
* try (Tensor t = Tensor.create(...)) {
* doSomethingWith(t);
* }
* }</pre>
*/
public final class Tensor<T> implements AutoCloseable {
/**
* Creates a Tensor from a Java object.
*
* <p>A {@code Tensor} is a multi-dimensional array of elements of a limited set of types. Not all
* Java objects can be converted to a {@code Tensor}. In particular, the argument {@code obj} must
* be either a primitive (float, double, int, long, boolean, byte) or a multi-dimensional array of
* one of those primitives. The argument {@code type} specifies how to interpret the first
* argument as a TensorFlow type. For example:
*
* <pre>{@code
* // Valid: A 64-bit integer scalar.
* Tensor<Long> s = Tensor.create(42L, Long.class);
*
* // Valid: A 3x2 matrix of floats.
* float[][] matrix = new float[3][2];
* Tensor<Float> m = Tensor.create(matrix, Float.class);
*
* // Invalid: Will throw an IllegalArgumentException as an arbitrary Object
* // does not fit into the TensorFlow type system.
* Tensor<?> o = Tensor.create(new Object())
*
* // Invalid: Will throw an IllegalArgumentException since there are
* // a differing number of elements in each row of this 2-D array.
* int[][] twoD = new int[2][];
* twoD[0] = new int[1];
* twoD[1] = new int[2];
* Tensor<Integer> x = Tensor.create(twoD, Integer.class);
* }</pre>
*
* {@link String}-typed Tensors are multi-dimensional arrays of arbitrary byte sequences, so can
* be initialized from arrays of {@code byte[]} elements. For example:
*
* <pre>{@code
* // Valid: A String tensor.
* Tensor<String> s = Tensor.create(new byte[]{1, 2, 3}, String.class);
*
* // Java Strings will need to be encoded into a byte-sequence.
* String mystring = "foo";
* Tensor<String> s = Tensor.create(mystring.getBytes("UTF-8"), String.class);
*
* // Valid: Matrix of String tensors.
* // Each element might have a different length.
* byte[][][] matrix = new byte[2][2][];
* matrix[0][0] = "this".getBytes("UTF-8");
* matrix[0][1] = "is".getBytes("UTF-8");
* matrix[1][0] = "a".getBytes("UTF-8");
* matrix[1][1] = "matrix".getBytes("UTF-8");
* Tensor<String> m = Tensor.create(matrix, String.class);
* }</pre>
*
* @param obj The object to convert to a {@code Tensor<T>}. Note that whether it is compatible
* with the type T is not checked by the type system. For type-safe creation of tensors, use
* {@link Tensors}.
* @param type The class object representing the type T.
* @throws IllegalArgumentException if {@code obj} is not compatible with the TensorFlow type
* system.
*/
@SuppressWarnings("unchecked")
public static <T> Tensor<T> create(Object obj, Class<T> type) {
DataType dtype = DataType.fromClass(type);
if (!objectCompatWithType(obj, dtype)) {
throw new IllegalArgumentException(
"DataType of object does not match T (expected "
+ dtype
+ ", got "
+ dataTypeOf(obj)
+ ")");
}
return (Tensor<T>) create(obj, dtype);
}
/**
* Creates a tensor from an object whose class is inspected to figure out what the underlying data
* type should be.
*
* @throws IllegalArgumentException if {@code obj} is not compatible with the TensorFlow type
* system.
*/
public static Tensor<?> create(Object obj) {
return create(obj, dataTypeOf(obj));
}
/**
* Create a Tensor of data type {@code dtype} from a Java object. Requires the parameter {@code T}
* to match {@code type}, but this condition is not checked.
*
* @param obj the object supplying the tensor data.
* @param dtype the data type of the tensor to create. It must be compatible with the run-time
* type of the object.
* @return the new tensor
*/
private static Tensor<?> create(Object obj, DataType dtype) {
@SuppressWarnings("rawtypes")
Tensor<?> t = new Tensor(dtype);
t.shapeCopy = new long[numDimensions(obj, dtype)];
fillShape(obj, 0, t.shapeCopy);
long nativeHandle;
if (t.dtype != DataType.STRING) {
int byteSize = elemByteSize(t.dtype) * numElements(t.shapeCopy);
nativeHandle = allocate(t.dtype.c(), t.shapeCopy, byteSize);
setValue(nativeHandle, obj);
} else if (t.shapeCopy.length != 0) {
nativeHandle = allocateNonScalarBytes(t.shapeCopy, (Object[]) obj);
} else {
nativeHandle = allocateScalarBytes((byte[]) obj);
}
t.nativeRef = new NativeReference(nativeHandle);
return t;
}
/**
* Create a {@link Integer} Tensor with data from the given buffer.
*
* <p>Creates a Tensor with the given shape by copying elements from the buffer (starting from its
* current position) into the tensor. For example, if {@code shape = {2,3} } (which represents a
* 2x3 matrix) then the buffer must have 6 elements remaining, which will be consumed by this
* method.
*
* @param shape the tensor shape.
* @param data a buffer containing the tensor data.
* @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
*/
public static Tensor<Integer> create(long[] shape, IntBuffer data) {
Tensor<Integer> t = allocateForBuffer(DataType.INT32, shape, data.remaining());
t.buffer().asIntBuffer().put(data);
return t;
}
/**
* Create a {@link Float} Tensor with data from the given buffer.
*
* <p>Creates a Tensor with the given shape by copying elements from the buffer (starting from its
* current position) into the tensor. For example, if {@code shape = {2,3} } (which represents a
* 2x3 matrix) then the buffer must have 6 elements remaining, which will be consumed by this
* method.
*
* @param shape the tensor shape.
* @param data a buffer containing the tensor data.
* @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
*/
public static Tensor<Float> create(long[] shape, FloatBuffer data) {
Tensor<Float> t = allocateForBuffer(DataType.FLOAT, shape, data.remaining());
t.buffer().asFloatBuffer().put(data);
return t;
}
/**
* Create a {@link Double} Tensor with data from the given buffer.
*
* <p>Creates a Tensor with the given shape by copying elements from the buffer (starting from its
* current position) into the tensor. For example, if {@code shape = {2,3} } (which represents a
* 2x3 matrix) then the buffer must have 6 elements remaining, which will be consumed by this
* method.
*
* @param shape the tensor shape.
* @param data a buffer containing the tensor data.
* @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
*/
public static Tensor<Double> create(long[] shape, DoubleBuffer data) {
Tensor<Double> t = allocateForBuffer(DataType.DOUBLE, shape, data.remaining());
t.buffer().asDoubleBuffer().put(data);
return t;
}
/**
* Create an {@link Long} Tensor with data from the given buffer.
*
* <p>Creates a Tensor with the given shape by copying elements from the buffer (starting from its
* current position) into the tensor. For example, if {@code shape = {2,3} } (which represents a
* 2x3 matrix) then the buffer must have 6 elements remaining, which will be consumed by this
* method.
*
* @param shape the tensor shape.
* @param data a buffer containing the tensor data.
* @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
*/
public static Tensor<Long> create(long[] shape, LongBuffer data) {
Tensor<Long> t = allocateForBuffer(DataType.INT64, shape, data.remaining());
t.buffer().asLongBuffer().put(data);
return t;
}
/**
* Create a Tensor of any type with data from the given buffer.
*
* <p>Creates a Tensor with the provided shape of any type where the tensor's data has been
* encoded into {@code data} as per the specification of the TensorFlow <a
* href="https://www.tensorflow.org/code/tensorflow/c/c_api.h">C
* API</a>.
*
* @param <T> the tensor element type
* @param type the tensor element type, represented as a class object.
* @param shape the tensor shape.
* @param data a buffer containing the tensor data.
* @throws IllegalArgumentException If the tensor datatype or shape is not compatible with the
* buffer
*/
public static <T> Tensor<T> create(Class<T> type, long[] shape, ByteBuffer data) {
@SuppressWarnings("unchecked")
Tensor<T> ret = (Tensor<T>) create(DataType.fromClass(type), shape, data);
return ret;
}
private static Tensor<?> create(DataType dtype, long[] shape, ByteBuffer data) {
int nremaining;
if (dtype != DataType.STRING) {
int elemBytes = elemByteSize(dtype);
if (data.remaining() % elemBytes != 0) {
throw new IllegalArgumentException(
String.format(
"ByteBuffer with %d bytes is not compatible with a %s Tensor (%d bytes/element)",
data.remaining(), dtype.toString(), elemBytes));
}
nremaining = data.remaining() / elemBytes;
} else {
nremaining = data.remaining();
}
Tensor<?> t = allocateForBuffer(dtype, shape, nremaining);
t.buffer().put(data);
return t;
}
/**
* Returns this Tensor object with the type {@code Tensor<U>}. This method is useful when given a
* value of type {@code Tensor<?>}.
*
* @param type any (non-null) array of the correct type.
* @throws IllegalArgumentException if the actual data type of this object does not match the type
* {@code U}.
*/
@SuppressWarnings("unchecked")
public <U> Tensor<U> expect(Class<U> type) {
DataType dt = DataType.fromClass(type);
if (!dt.equals(dtype)) {
throw new IllegalArgumentException(
"Cannot cast from tensor of " + dtype + " to tensor of " + dt);
}
return ((Tensor<U>) this);
}
// Helper function to allocate a Tensor for the create() methods that create a Tensor from
// a java.nio.Buffer.
// Requires: dataType matches T
private static <T> Tensor<T> allocateForBuffer(DataType dataType, long[] shape, int nBuffered) {
final int nflattened = numElements(shape);
int nbytes = 0;
if (dataType != DataType.STRING) {
if (nBuffered != nflattened) {
throw incompatibleBuffer(nBuffered, shape);
}
nbytes = nflattened * elemByteSize(dataType);
} else {
// DT_STRING tensor encoded in a ByteBuffer.
nbytes = nBuffered;
}
Tensor<T> t = new Tensor<T>(dataType);
t.shapeCopy = Arrays.copyOf(shape, shape.length);
long nativeHandle = allocate(t.dtype.c(), t.shapeCopy, nbytes);
t.nativeRef = new NativeReference(nativeHandle);
return t;
}
/**
* Release resources associated with the Tensor.
*
* <p><b>WARNING:</b>This must be invoked for all tensors that were not been produced by an eager
* operation or memory will be leaked.
*
* <p>The Tensor object is no longer usable after {@code close} returns.
*/
@Override
public void close() {
nativeRef.release();
}
/** Returns the {@link DataType} of elements stored in the Tensor. */
public DataType dataType() {
return dtype;
}
/**
* Returns the number of dimensions (sometimes referred to as <a
* href="https://www.tensorflow.org/resources/dims_types.html#rank">rank</a>) of the Tensor.
*
* <p>Will be 0 for a scalar, 1 for a vector, 2 for a matrix, 3 for a 3-dimensional tensor etc.
*/
public int numDimensions() {
return shapeCopy.length;
}
/** Returns the size, in bytes, of the tensor data. */
public int numBytes() {
return buffer().remaining();
}
/** Returns the number of elements in a flattened (1-D) view of the tensor. */
public int numElements() {
return numElements(shapeCopy);
}
/**
* Returns the <a href="https://www.tensorflow.org/resources/dims_types.html#shape">shape</a> of
* the Tensor, i.e., the sizes of each dimension.
*
* @return an array where the i-th element is the size of the i-th dimension of the tensor.
*/
public long[] shape() {
return shapeCopy;
}
/**
* Returns the value in a scalar {@link Float} tensor.
*
* @throws IllegalArgumentException if the Tensor does not represent a float scalar.
*/
public float floatValue() {
return scalarFloat(getNativeHandle());
}
/**
* Returns the value in a scalar {@link Double} tensor.
*
* @throws IllegalArgumentException if the Tensor does not represent a double scalar.
*/
public double doubleValue() {
return scalarDouble(getNativeHandle());
}
/**
* Returns the value in a scalar {@link Integer} tensor.
*
* @throws IllegalArgumentException if the Tensor does not represent a int scalar.
*/
public int intValue() {
return scalarInt(getNativeHandle());
}
/**
* Returns the value in a scalar {@link Long} tensor.
*
* @throws IllegalArgumentException if the Tensor does not represent a long scalar.
*/
public long longValue() {
return scalarLong(getNativeHandle());
}
/**
* Returns the value in a scalar {@link Boolean} tensor.
*
* @throws IllegalArgumentException if the Tensor does not represent a boolean scalar.
*/
public boolean booleanValue() {
return scalarBoolean(getNativeHandle());
}
/**
* Returns the value in a scalar {@link String} tensor.
*
* @throws IllegalArgumentException if the Tensor does not represent a boolean scalar.
*/
public byte[] bytesValue() {
return scalarBytes(getNativeHandle());
}
/**
* Copies the contents of the tensor to {@code dst} and returns {@code dst}.
*
* <p>For non-scalar tensors, this method copies the contents of the underlying tensor to a Java
* array. For scalar tensors, use one of {@link #bytesValue()}, {@link #floatValue()}, {@link
* #doubleValue()}, {@link #intValue()}, {@link #longValue()} or {@link #booleanValue()} instead.
* The type and shape of {@code dst} must be compatible with the tensor. For example:
*
* <pre>{@code
* int matrix[2][2] = {{1,2},{3,4}};
* try(Tensor t = Tensor.create(matrix)) {
* // Succeeds and prints "3"
* int[][] copy = new int[2][2];
* System.out.println(t.copyTo(copy)[1][0]);
*
* // Throws IllegalArgumentException since the shape of dst does not match the shape of t.
* int[][] dst = new int[4][1];
* t.copyTo(dst);
* }
* }</pre>
*
* @throws IllegalArgumentException if the tensor is a scalar or if {@code dst} is not compatible
* with the tensor (for example, mismatched data types or shapes).
*/
public <U> U copyTo(U dst) {
throwExceptionIfTypeIsIncompatible(dst);
readNDArray(getNativeHandle(), dst);
return dst;
}
/**
* Write the data of a {@link Integer} tensor into the given buffer.
*
* <p>Copies {@code numElements()} elements to the buffer.
*
* @param dst the destination buffer
* @throws BufferOverflowException If there is insufficient space in the given buffer for the data
* in this tensor
* @throws IllegalArgumentException If the tensor data type is not {@link Integer}
*/
public void writeTo(IntBuffer dst) {
if (dtype != DataType.INT32) {
throw incompatibleBuffer(dst, dtype);
}
ByteBuffer src = buffer();
dst.put(src.asIntBuffer());
}
/**
* Write the data of a {@link Float} tensor into the given buffer.
*
* <p>Copies {@code numElements()} elements to the buffer.
*
* @param dst the destination buffer
* @throws BufferOverflowException If there is insufficient space in the given buffer for the data
* in this tensor
* @throws IllegalArgumentException If the tensor datatype is not {@link Float}
*/
public void writeTo(FloatBuffer dst) {
if (dtype != DataType.FLOAT) {
throw incompatibleBuffer(dst, dtype);
}
ByteBuffer src = buffer();
dst.put(src.asFloatBuffer());
}
/**
* Write the data of a {@link Double} tensor into the given buffer.
*
* <p>Copies {@code numElements()} elements to the buffer.
*
* @param dst the destination buffer
* @throws BufferOverflowException If there is insufficient space in the given buffer for the data
* in this tensor
* @throws IllegalArgumentException If the tensor datatype is not {@link Double}
*/
public void writeTo(DoubleBuffer dst) {
if (dtype != DataType.DOUBLE) {
throw incompatibleBuffer(dst, dtype);
}
ByteBuffer src = buffer();
dst.put(src.asDoubleBuffer());
}
/**
* Write the data of a {@link Long} tensor into the given buffer.
*
* <p>Copies {@code numElements()} elements to the buffer.
*
* @param dst the destination buffer
* @throws BufferOverflowException If there is insufficient space in the given buffer for the data
* in this tensor
* @throws IllegalArgumentException If the tensor datatype is not {@link Long}
*/
public void writeTo(LongBuffer dst) {
if (dtype != DataType.INT64) {
throw incompatibleBuffer(dst, dtype);
}
ByteBuffer src = buffer();
dst.put(src.asLongBuffer());
}
/**
* Write the tensor data into the given buffer.
*
* <p>Copies {@code numBytes()} bytes to the buffer in native byte order for primitive types.
*
* @param dst the destination buffer
* @throws BufferOverflowException If there is insufficient space in the given buffer for the data
* in this tensor
*/
public void writeTo(ByteBuffer dst) {
ByteBuffer src = buffer();
dst.put(src);
}
/** Returns a string describing the type and shape of the Tensor. */
@Override
public String toString() {
return String.format("%s tensor with shape %s", dtype.toString(), Arrays.toString(shape()));
}
/**
* Create a Tensor object from a handle to the C TF_Tensor object.
*
* <p>Takes ownership of the handle.
*/
static Tensor<?> fromHandle(long handle) {
@SuppressWarnings("rawtypes")
Tensor<?> t = new Tensor(DataType.fromC(dtype(handle)));
t.shapeCopy = shape(handle);
t.nativeRef = new NativeReference(handle);
return t;
}
/**
* Create an eager Tensor object from a handle to the C TF_Tensor object.
*
* <p>Takes ownership of the handle.
*/
static Tensor<?> fromHandle(long handle, EagerSession session) {
Tensor<?> t = fromHandle(handle);
t.nativeRef.eager(session, t);
return t;
}
long getNativeHandle() {
return nativeRef.tensorHandle;
}
private NativeReference nativeRef = null;
private final DataType dtype;
private long[] shapeCopy = null;
private Tensor(DataType t) {
dtype = t;
}
private ByteBuffer buffer() {
return buffer(getNativeHandle()).order(ByteOrder.nativeOrder());
}
private static IllegalArgumentException incompatibleBuffer(Buffer buf, DataType dataType) {
return new IllegalArgumentException(
String.format("cannot use %s with Tensor of type %s", buf.getClass().getName(), dataType));
}
private static IllegalArgumentException incompatibleBuffer(int numElements, long[] shape) {
return new IllegalArgumentException(
String.format(
"buffer with %d elements is not compatible with a Tensor with shape %s",
numElements, Arrays.toString(shape)));
}
private static int numElements(long[] shape) {
// assumes a fully-known shape
int n = 1;
for (int i = 0; i < shape.length; i++) {
n *= (int) shape[i];
}
return n;
}
private static int elemByteSize(DataType dataType) {
int size = dataType.byteSize();
if (size < 0) {
throw new IllegalArgumentException("STRING tensors do not have a fixed element size");
}
return size;
}
private static void throwExceptionIfNotByteOfByteArrays(Object array) {
if (!array.getClass().getName().equals("[[B")) {
throw new IllegalArgumentException(
"object cannot be converted to a Tensor as it includes an array with null elements");
}
}
/**
* Reference to the underlying native tensor
*
* <p>Tensors are commonly allocated in a `try-with-resources` statement, where they get
* automatically released after executing the last line of the `try` block they were declared in.
*
* <p>They can also be attached to an eager session, where in this case their lifetime ends either
* when this session is closed or when the Tensor instance is no longer referenced and have been
* garbage-collected.
*
* <p>This helper class wraps the tensor native handle and support both situations; If an eager
* reference to the tensor exists, it will take care of releasing the tensor at the end of its
* life. If the tensor is being explicitly closed before this happens, it will take cake of
* clearing its association with any eager session before cleaning up the resources.
*/
private static class NativeReference {
/** Attaches this reference to an eager session */
private class EagerReference extends EagerSession.NativeReference {
EagerReference(EagerSession session, Tensor<?> tensor) {
super(session, tensor);
}
@Override
void delete() {
// Mark this eager reference as cleared since it has been deleted by the session
NativeReference.this.eagerRef = null;
NativeReference.this.release();
}
}
NativeReference(long tensorHandle) {
this.tensorHandle = tensorHandle;
}
void eager(EagerSession session, Tensor<?> tensor) {
if (eagerRef != null) {
throw new IllegalStateException("The tensor is already attached to an eager session");
}
eagerRef = new EagerReference(session, tensor);
}
synchronized void release() {
if (tensorHandle != 0L) {
// Clear any remaining eager reference to this tensor
if (eagerRef != null) {
eagerRef.clear();
eagerRef = null;
}
Tensor.delete(tensorHandle);
tensorHandle = 0L;
}
}
private long tensorHandle;
private EagerReference eagerRef;
}
private static HashMap<Class<?>, DataType> classDataTypes = new HashMap<>();
static {
classDataTypes.put(int.class, DataType.INT32);
classDataTypes.put(Integer.class, DataType.INT32);
classDataTypes.put(long.class, DataType.INT64);
classDataTypes.put(Long.class, DataType.INT64);
classDataTypes.put(float.class, DataType.FLOAT);
classDataTypes.put(Float.class, DataType.FLOAT);
classDataTypes.put(double.class, DataType.DOUBLE);
classDataTypes.put(Double.class, DataType.DOUBLE);
classDataTypes.put(byte.class, DataType.STRING);
classDataTypes.put(Byte.class, DataType.STRING);
classDataTypes.put(boolean.class, DataType.BOOL);
classDataTypes.put(Boolean.class, DataType.BOOL);
}
/** The class for the data type to which Java object o corresponds. */
private static Class<?> baseObjType(Object o) {
Class<?> c = o.getClass();
while (c.isArray()) {
c = c.getComponentType();
}
return c;
}
/**
* The default TensorFlow data type to which Java object o corresponds. Some Java objects
* represent more than one TensorFlow data type; for example, 'byte' can represent both {@code
* uint8} and {@code string}, with the latter being the default interpretation.
*/
private static DataType dataTypeOf(Object o) {
Class<?> c = baseObjType(o);
return dataTypeFromClass(c);
}
private static DataType dataTypeFromClass(Class<?> c) {
DataType ret = classDataTypes.get(c);
if (ret != null) {
return ret;
}
throw new IllegalArgumentException("cannot create Tensors of type " + c.getName());
}
/**
* Return the number of dimensions of the tensor that object {@code o} represents as a tensor
* whose datatype is {@code dtype}. Normally this is the same as the number of dimensions of o
* itself, but is one smaller for tensors of strings.
*
* @param o The object to inspect. It must be a valid representation of the given data type.
* @param dtype The expected data type of the tensor.
*/
private static int numDimensions(Object o, DataType dtype) {
int ret = numArrayDimensions(o);
if (dtype == DataType.STRING && ret > 0) {
return ret - 1;
}
return ret;
}
/** Returns the number of dimensions of the array object o. Returns 0 if o is not an array. */
private static int numArrayDimensions(Object o) {
Class<?> c = o.getClass();
int i = 0;
while (c.isArray()) {
c = c.getComponentType();
i++;
}
return i;
}
/**
* Fills in the remaining entries in the shape array starting from position {@code dim} with the
* dimension sizes of the multidimensional array o. Checks that all arrays reachable from o have
* sizes consistent with the filled-in shape, throwing IllegalArgumentException otherwise.
*/
private static void fillShape(Object o, int dim, long[] shape) {
if (shape == null || dim == shape.length) {
return;
}
final int len = Array.getLength(o);
if (len == 0) {
throw new IllegalArgumentException("cannot create Tensors with a 0 dimension");
}
if (shape[dim] == 0) {
shape[dim] = len;
} else if (shape[dim] != len) {
throw new IllegalArgumentException(
String.format("mismatched lengths (%d and %d) in dimension %d", shape[dim], len, dim));
}
for (int i = 0; i < len; ++i) {
fillShape(Array.get(o, i), dim + 1, shape);
}
}
/** Returns whether the object {@code obj} can represent a tensor with data type {@code dtype}. */
private static boolean objectCompatWithType(Object obj, DataType dtype) {
Class<?> c = baseObjType(obj);
DataType dto = dataTypeFromClass(c);
int nd = numDimensions(obj, dto);
if (!c.isPrimitive() && c != String.class && nd != 0) {
throw new IllegalArgumentException(
"cannot create non-scalar Tensors from arrays of boxed values");
}
if (dto.equals(dtype)) {
return true;
}
if (dto == DataType.STRING && dtype == DataType.UINT8) {
return true;
}
return false;
}
private void throwExceptionIfTypeIsIncompatible(Object o) {
final int rank = numDimensions();
final int oRank = numDimensions(o, dtype);
if (oRank != rank) {
throw new IllegalArgumentException(
String.format(
"cannot copy Tensor with %d dimensions into an object with %d", rank, oRank));
}
if (!objectCompatWithType(o, dtype)) {
throw new IllegalArgumentException(
String.format(
"cannot copy Tensor with DataType %s into an object of type %s",
dtype.toString(), o.getClass().getName()));
}
long[] oShape = new long[rank];
fillShape(o, 0, oShape);
for (int i = 0; i < oShape.length; ++i) {
if (oShape[i] != shape()[i]) {
throw new IllegalArgumentException(
String.format(
"cannot copy Tensor with shape %s into object with shape %s",
Arrays.toString(shape()), Arrays.toString(oShape)));
}
}
}
private static native long allocate(int dtype, long[] shape, long byteSize);
private static native long allocateScalarBytes(byte[] value);
private static native long allocateNonScalarBytes(long[] shape, Object[] value);
private static native void delete(long handle);
private static native ByteBuffer buffer(long handle);
private static native int dtype(long handle);
private static native long[] shape(long handle);
private static native void setValue(long handle, Object value);
private static native float scalarFloat(long handle);
private static native double scalarDouble(long handle);
private static native int scalarInt(long handle);
private static native long scalarLong(long handle);
private static native boolean scalarBoolean(long handle);
private static native byte[] scalarBytes(long handle);
private static native void readNDArray(long handle, Object value);
static {
TensorFlow.init();
}
}
tensorflow/java/src/main/java/org/tensorflow/TensorFlow.java
+84
View File
@@ -0,0 +1,84 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
/** Static utility methods describing the TensorFlow runtime. */
public final class TensorFlow {
/** Returns the version of the underlying TensorFlow runtime. */
public static native String version();
/**
* All the TensorFlow operations available in this address space.
*
* @return A serialized representation of an <a
* href="https://www.tensorflow.org/code/tensorflow/core/framework/op_def.proto">OpList</a>
* protocol buffer, which lists all the available TensorFlow operations.
*/
public static native byte[] registeredOpList();
/**
* Load the dynamic library in filename and register the operations and kernels present in that
* library.
*
* @param filename Path of the dynamic library containing operations and kernels to load.
* @return Serialized bytes of the <a
* href="https://www.tensorflow.org/code/tensorflow/core/framework/op_def.proto">OpList</a>
* protocol buffer message defining the operations defined in the library.
* @throws UnsatisfiedLinkError if filename cannot be loaded.
*/
public static byte[] loadLibrary(String filename) {
long h = 0;
try {
h = libraryLoad(filename);
} catch (RuntimeException e) {
throw new UnsatisfiedLinkError(e.getMessage());
}
try {
return libraryOpList(h);
} finally {
libraryDelete(h);
}
}
private static native long libraryLoad(String filename);
private static native void libraryDelete(long handle);
private static native byte[] libraryOpList(long handle);
private TensorFlow() {}
/** Load the TensorFlow runtime C library. */
static void init() {
try {
NativeLibrary.load();
} catch (Exception e) {
/*
* This code is called during static initialization of this and of other classes.
* If this fails then a NoClassDefFoundError is thrown however this does not
* include a cause. Printing the exception manually here ensures that the
* necessary information to fix the problem is available.
*/
System.err.println("Failed to load TensorFlow native library");
e.printStackTrace();
throw e;
}
}
static {
init();
}
}
tensorflow/java/src/main/java/org/tensorflow/TensorFlowException.java
+23
View File
@@ -0,0 +1,23 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
/** Unchecked exception thrown when executing TensorFlow Graphs. */
public final class TensorFlowException extends RuntimeException {
TensorFlowException(String message) {
super(message);
}
}
tensorflow/java/src/main/java/org/tensorflow/Tensors.java
+447
View File
@@ -0,0 +1,447 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import static java.nio.charset.StandardCharsets.UTF_8;
/** Type-safe factory methods for creating {@link org.tensorflow.Tensor} objects. */
public final class Tensors {
private Tensors() {}
/**
* Creates a scalar String tensor using the default, UTF-8 encoding.
*
* @param data The string to put into the new scalar tensor.
*/
public static Tensor<String> create(String data) {
return Tensor.create(data.getBytes(UTF_8), String.class);
}
/**
* Creates a scalar String tensor using a specified encoding.
*
* @param charset The encoding from String to bytes.
* @param data The string to put into the new scalar tensor.
*/
public static Tensor<String> create(String data, java.nio.charset.Charset charset) {
return Tensor.create(data.getBytes(charset), String.class);
}
/**
* Creates a scalar tensor containing a single {@code float} element.
*
* @param data The value to put into the new scalar tensor.
*/
public static Tensor<Float> create(float data) {
return Tensor.create(data, Float.class);
}
/**
* Creates a rank-1 tensor of {@code float} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Float> create(float[] data) {
return Tensor.create(data, Float.class);
}
/**
* Creates a rank-2 tensor of {@code float} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Float> create(float[][] data) {
return Tensor.create(data, Float.class);
}
/**
* Creates a rank-3 tensor of {@code float} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Float> create(float[][][] data) {
return Tensor.create(data, Float.class);
}
/**
* Creates a rank-4 tensor of {@code float} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Float> create(float[][][][] data) {
return Tensor.create(data, Float.class);
}
/**
* Creates a rank-5 tensor of {@code float} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Float> create(float[][][][][] data) {
return Tensor.create(data, Float.class);
}
/**
* Creates a rank-6 tensor of {@code float} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Float> create(float[][][][][][] data) {
return Tensor.create(data, Float.class);
}
/**
* Creates a scalar tensor containing a single {@code double} element.
*
* @param data The value to put into the new scalar tensor.
*/
public static Tensor<Double> create(double data) {
return Tensor.create(data, Double.class);
}
/**
* Creates a rank-1 tensor of {@code double} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Double> create(double[] data) {
return Tensor.create(data, Double.class);
}
/**
* Creates a rank-2 tensor of {@code double} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Double> create(double[][] data) {
return Tensor.create(data, Double.class);
}
/**
* Creates a rank-3 tensor of {@code double} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Double> create(double[][][] data) {
return Tensor.create(data, Double.class);
}
/**
* Creates a rank-4 tensor of {@code double} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Double> create(double[][][][] data) {
return Tensor.create(data, Double.class);
}
/**
* Creates a rank-5 tensor of {@code double} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Double> create(double[][][][][] data) {
return Tensor.create(data, Double.class);
}
/**
* Creates a rank-6 tensor of {@code double} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Double> create(double[][][][][][] data) {
return Tensor.create(data, Double.class);
}
/**
* Creates a scalar tensor containing a single {@code int} element.
*
* @param data The value to put into the new scalar tensor.
*/
public static Tensor<Integer> create(int data) {
return Tensor.create(data, Integer.class);
}
/**
* Creates a rank-1 tensor of {@code int} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Integer> create(int[] data) {
return Tensor.create(data, Integer.class);
}
/**
* Creates a rank-2 tensor of {@code int} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Integer> create(int[][] data) {
return Tensor.create(data, Integer.class);
}
/**
* Creates a rank-3 tensor of {@code int} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Integer> create(int[][][] data) {
return Tensor.create(data, Integer.class);
}
/**
* Creates a rank-4 tensor of {@code int} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Integer> create(int[][][][] data) {
return Tensor.create(data, Integer.class);
}
/**
* Creates a rank-5 tensor of {@code int} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Integer> create(int[][][][][] data) {
return Tensor.create(data, Integer.class);
}
/**
* Creates a rank-6 tensor of {@code int} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Integer> create(int[][][][][][] data) {
return Tensor.create(data, Integer.class);
}
/**
* Creates a scalar tensor containing a single {@code byte} element.
*
* @param data An array containing the data to put into the new tensor. String elements are
* sequences of bytes from the last array dimension.
*/
public static Tensor<String> create(byte[] data) {
return Tensor.create(data, String.class);
}
/**
* Creates a rank-1 tensor of {@code byte} elements.
*
* @param data An array containing the data to put into the new tensor. String elements are
* sequences of bytes from the last array dimension.
*/
public static Tensor<String> create(byte[][] data) {
return Tensor.create(data, String.class);
}
/**
* Creates a rank-2 tensor of {@code byte} elements.
*
* @param data An array containing the data to put into the new tensor. String elements are
* sequences of bytes from the last array dimension.
*/
public static Tensor<String> create(byte[][][] data) {
return Tensor.create(data, String.class);
}
/**
* Creates a rank-3 tensor of {@code byte} elements.
*
* @param data An array containing the data to put into the new tensor. String elements are
* sequences of bytes from the last array dimension.
*/
public static Tensor<String> create(byte[][][][] data) {
return Tensor.create(data, String.class);
}
/**
* Creates a rank-4 tensor of {@code byte} elements.
*
* @param data An array containing the data to put into the new tensor. String elements are
* sequences of bytes from the last array dimension.
*/
public static Tensor<String> create(byte[][][][][] data) {
return Tensor.create(data, String.class);
}
/**
* Creates a rank-5 tensor of {@code byte} elements.
*
* @param data An array containing the data to put into the new tensor. String elements are
* sequences of bytes from the last array dimension.
*/
public static Tensor<String> create(byte[][][][][][] data) {
return Tensor.create(data, String.class);
}
/**
* Creates a scalar tensor containing a single {@code long} element.
*
* @param data The value to put into the new scalar tensor.
*/
public static Tensor<Long> create(long data) {
return Tensor.create(data, Long.class);
}
/**
* Creates a rank-1 tensor of {@code long} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Long> create(long[] data) {
return Tensor.create(data, Long.class);
}
/**
* Creates a rank-2 tensor of {@code long} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Long> create(long[][] data) {
return Tensor.create(data, Long.class);
}
/**
* Creates a rank-3 tensor of {@code long} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Long> create(long[][][] data) {
return Tensor.create(data, Long.class);
}
/**
* Creates a rank-4 tensor of {@code long} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Long> create(long[][][][] data) {
return Tensor.create(data, Long.class);
}
/**
* Creates a rank-5 tensor of {@code long} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Long> create(long[][][][][] data) {
return Tensor.create(data, Long.class);
}
/**
* Creates a rank-6 tensor of {@code long} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Long> create(long[][][][][][] data) {
return Tensor.create(data, Long.class);
}
/**
* Creates a scalar tensor containing a single {@code boolean} element.
*
* @param data The value to put into the new scalar tensor.
*/
public static Tensor<Boolean> create(boolean data) {
return Tensor.create(data, Boolean.class);
}
/**
* Creates a rank-1 tensor of {@code boolean} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Boolean> create(boolean[] data) {
return Tensor.create(data, Boolean.class);
}
/**
* Creates a rank-2 tensor of {@code boolean} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Boolean> create(boolean[][] data) {
return Tensor.create(data, Boolean.class);
}
/**
* Creates a rank-3 tensor of {@code boolean} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Boolean> create(boolean[][][] data) {
return Tensor.create(data, Boolean.class);
}
/**
* Creates a rank-4 tensor of {@code boolean} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Boolean> create(boolean[][][][] data) {
return Tensor.create(data, Boolean.class);
}
/**
* Creates a rank-5 tensor of {@code boolean} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Boolean> create(boolean[][][][][] data) {
return Tensor.create(data, Boolean.class);
}
/**
* Creates a rank-6 tensor of {@code boolean} elements.
*
* @param data An array containing the values to put into the new tensor. The dimensions of the
* new tensor will match those of the array.
*/
public static Tensor<Boolean> create(boolean[][][][][][] data) {
return Tensor.create(data, Boolean.class);
}
}
tensorflow/java/src/main/java/org/tensorflow/examples/.gitignore
+3
View File
@@ -0,0 +1,3 @@
# .class files generated when building examples using javac
# as described in README.md
*.class
tensorflow/java/src/main/java/org/tensorflow/examples/BUILD
+17
View File
@@ -0,0 +1,17 @@
# Description:
# TensorFlow Java examples.
load("@rules_java//java:defs.bzl", "java_binary")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//visibility:private"],
licenses = ["notice"],
)
java_binary(
name = "label_image",
srcs = ["LabelImage.java"],
main_class = "org.tensorflow.examples.LabelImage",
deps = ["//tensorflow/java:tensorflow"],
)
tensorflow/java/src/main/java/org/tensorflow/examples/LabelImage.java
+235
View File
@@ -0,0 +1,235 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.examples;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import org.tensorflow.DataType;
import org.tensorflow.Graph;
import org.tensorflow.Output;
import org.tensorflow.Session;
import org.tensorflow.Tensor;
import org.tensorflow.TensorFlow;
import org.tensorflow.types.UInt8;
/** Sample use of the TensorFlow Java API to label images using a pre-trained model. */
public class LabelImage {
private static void printUsage(PrintStream s) {
final String url =
"https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip";
s.println(
"Java program that uses a pre-trained Inception model (http://arxiv.org/abs/1512.00567)");
s.println("to label JPEG images.");
s.println("TensorFlow version: " + TensorFlow.version());
s.println();
s.println("Usage: label_image <model dir> <image file>");
s.println();
s.println("Where:");
s.println("<model dir> is a directory containing the unzipped contents of the inception model");
s.println(" (from " + url + ")");
s.println("<image file> is the path to a JPEG image file");
}
public static void main(String[] args) {
if (args.length != 2) {
printUsage(System.err);
System.exit(1);
}
String modelDir = args[0];
String imageFile = args[1];
byte[] graphDef = readAllBytesOrExit(Paths.get(modelDir, "tensorflow_inception_graph.pb"));
List<String> labels =
readAllLinesOrExit(Paths.get(modelDir, "imagenet_comp_graph_label_strings.txt"));
byte[] imageBytes = readAllBytesOrExit(Paths.get(imageFile));
try (Tensor<Float> image = constructAndExecuteGraphToNormalizeImage(imageBytes)) {
float[] labelProbabilities = executeInceptionGraph(graphDef, image);
int bestLabelIdx = maxIndex(labelProbabilities);
System.out.println(
String.format("BEST MATCH: %s (%.2f%% likely)",
labels.get(bestLabelIdx),
labelProbabilities[bestLabelIdx] * 100f));
}
}
private static Tensor<Float> constructAndExecuteGraphToNormalizeImage(byte[] imageBytes) {
try (Graph g = new Graph()) {
GraphBuilder b = new GraphBuilder(g);
// Some constants specific to the pre-trained model at:
// https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip
//
// - The model was trained with images scaled to 224x224 pixels.
// - The colors, represented as R, G, B in 1-byte each were converted to
// float using (value - Mean)/Scale.
final int H = 224;
final int W = 224;
final float mean = 117f;
final float scale = 1f;
// Since the graph is being constructed once per execution here, we can use a constant for the
// input image. If the graph were to be re-used for multiple input images, a placeholder would
// have been more appropriate.
final Output<String> input = b.constant("input", imageBytes);
final Output<Float> output =
b.div(
b.sub(
b.resizeBilinear(
b.expandDims(
b.cast(b.decodeJpeg(input, 3), Float.class),
b.constant("make_batch", 0)),
b.constant("size", new int[] {H, W})),
b.constant("mean", mean)),
b.constant("scale", scale));
try (Session s = new Session(g)) {
// Generally, there may be multiple output tensors, all of them must be closed to prevent resource leaks.
return s.runner().fetch(output.op().name()).run().get(0).expect(Float.class);
}
}
}
private static float[] executeInceptionGraph(byte[] graphDef, Tensor<Float> image) {
try (Graph g = new Graph()) {
g.importGraphDef(graphDef);
try (Session s = new Session(g);
// Generally, there may be multiple output tensors, all of them must be closed to prevent resource leaks.
Tensor<Float> result =
s.runner().feed("input", image).fetch("output").run().get(0).expect(Float.class)) {
final long[] rshape = result.shape();
if (result.numDimensions() != 2 || rshape[0] != 1) {
throw new RuntimeException(
String.format(
"Expected model to produce a [1 N] shaped tensor where N is the number of labels, instead it produced one with shape %s",
Arrays.toString(rshape)));
}
int nlabels = (int) rshape[1];
return result.copyTo(new float[1][nlabels])[0];
}
}
}
private static int maxIndex(float[] probabilities) {
int best = 0;
for (int i = 1; i < probabilities.length; ++i) {
if (probabilities[i] > probabilities[best]) {
best = i;
}
}
return best;
}
private static byte[] readAllBytesOrExit(Path path) {
try {
return Files.readAllBytes(path);
} catch (IOException e) {
System.err.println("Failed to read [" + path + "]: " + e.getMessage());
System.exit(1);
}
return null;
}
private static List<String> readAllLinesOrExit(Path path) {
try {
return Files.readAllLines(path, Charset.forName("UTF-8"));
} catch (IOException e) {
System.err.println("Failed to read [" + path + "]: " + e.getMessage());
System.exit(0);
}
return null;
}
// In the fullness of time, equivalents of the methods of this class should be auto-generated from
// the OpDefs linked into libtensorflow_jni.so. That would match what is done in other languages
// like Python, C++ and Go.
static class GraphBuilder {
GraphBuilder(Graph g) {
this.g = g;
}
Output<Float> div(Output<Float> x, Output<Float> y) {
return binaryOp("Div", x, y);
}
<T> Output<T> sub(Output<T> x, Output<T> y) {
return binaryOp("Sub", x, y);
}
<T> Output<Float> resizeBilinear(Output<T> images, Output<Integer> size) {
return binaryOp3("ResizeBilinear", images, size);
}
<T> Output<T> expandDims(Output<T> input, Output<Integer> dim) {
return binaryOp3("ExpandDims", input, dim);
}
<T, U> Output<U> cast(Output<T> value, Class<U> type) {
DataType dtype = DataType.fromClass(type);
return g.opBuilder("Cast", "Cast")
.addInput(value)
.setAttr("DstT", dtype)
.build()
.<U>output(0);
}
Output<UInt8> decodeJpeg(Output<String> contents, long channels) {
return g.opBuilder("DecodeJpeg", "DecodeJpeg")
.addInput(contents)
.setAttr("channels", channels)
.build()
.<UInt8>output(0);
}
<T> Output<T> constant(String name, Object value, Class<T> type) {
try (Tensor<T> t = Tensor.<T>create(value, type)) {
return g.opBuilder("Const", name)
.setAttr("dtype", DataType.fromClass(type))
.setAttr("value", t)
.build()
.<T>output(0);
}
}
Output<String> constant(String name, byte[] value) {
return this.constant(name, value, String.class);
}
Output<Integer> constant(String name, int value) {
return this.constant(name, value, Integer.class);
}
Output<Integer> constant(String name, int[] value) {
return this.constant(name, value, Integer.class);
}
Output<Float> constant(String name, float value) {
return this.constant(name, value, Float.class);
}
private <T> Output<T> binaryOp(String type, Output<T> in1, Output<T> in2) {
return g.opBuilder(type, type).addInput(in1).addInput(in2).build().<T>output(0);
}
private <T, U, V> Output<T> binaryOp3(String type, Output<U> in1, Output<V> in2) {
return g.opBuilder(type, type).addInput(in1).addInput(in2).build().<T>output(0);
}
private Graph g;
}
}
tensorflow/java/src/main/java/org/tensorflow/op/NameScope.java
+146
View File
@@ -0,0 +1,146 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.op;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;
/**
* A class to manage scoped (hierarchical) names for operators.
*
* <p>{@code NameScope} manages hierarchical names where each component in the hierarchy is
* separated by a forward slash {@code '/'}. For instance, {@code nn/Const_72} or {@code
* nn/gradient/assign/init}. Each scope is a subtree in this hierarchy.
*
* <p>Use {@code NameScope} to group related operations within a hierarchy, which for example lets
* tensorboard coalesce nodes for better graph visualizations.
*
* <p>This class is package private, user code creates {@link Scope} which internally delegates
* calls to an underlying {@code NameScope}.
*
* <p>This class is <b>not</b> thread-safe.
*/
final class NameScope {
NameScope withSubScope(String scopeName) {
checkPattern(NAME_REGEX, scopeName);
// Override with opName if it exists.
String actualName = (opName != null) ? opName : scopeName;
String newPrefix = fullyQualify(makeUnique(actualName));
return new NameScope(newPrefix, null, null);
}
NameScope withName(String name) {
checkPattern(NAME_REGEX, name);
// All context except for the opName is shared with the new scope.
return new NameScope(opPrefix, name, ids);
}
String makeOpName(String name) {
checkPattern(NAME_REGEX, name);
// Override with opName if it exists.
String actualName = (opName != null) ? opName : name;
return fullyQualify(makeUnique(actualName));
}
/**
* Create a new, root-level namescope.
*
* <p>A root-level namescope generates operator names with no components, like {@code Const_72}
* and {@code result}.
*/
NameScope() {
this(null, null, null);
}
private NameScope(String opPrefix, String opName, Map<String, Integer> ids) {
this.opPrefix = opPrefix;
this.opName = opName;
if (ids != null) {
this.ids = ids;
} else {
this.ids = new HashMap<String, Integer>();
}
}
// Generate a unique name, different from existing ids.
//
// ids is a map from id to integer, representing a counter of the
// number of previous requests to generate a unique name for the
// given id.
//
// For instance, the first use of makeUnique("a") adds "a" -> 1
// to ids and returns "a".
//
// The second use of makeUnique("a") updates ids to "a" -> 2
// and returns "a_1", and so on.
private String makeUnique(String id) {
if (!ids.containsKey(id)) {
ids.put(id, 1);
return id;
} else {
int cur = ids.get(id);
ids.put(id, cur + 1);
return String.format("%s_%d", id, cur);
}
}
private String fullyQualify(String name) {
if (opPrefix != null) {
return String.format("%s/%s", opPrefix, name);
} else {
return name;
}
}
// If opPrefix is non-null, it is a prefix applied to all names
// created by this instance.
private final String opPrefix;
// If opName is non-null, it is used to derive the unique name
// for operators rather than the provided default name.
private final String opName;
// NameScope generates unique names by appending a numeric suffix if
// needed. This is a map containing names already created by this
// instance mapped to the next available numeric suffix for it.
private final Map<String, Integer> ids;
private static void checkPattern(Pattern pattern, String name) {
if (name == null) {
throw new IllegalArgumentException("Names cannot be null");
}
if (!pattern.matcher(name).matches()) {
throw new IllegalArgumentException(
String.format(
"invalid name: '%s' does not match the regular expression %s",
name, NAME_REGEX.pattern()));
}
}
// The constraints for operator and scope names originate from restrictions on node names
// noted in the proto definition core/framework/node_def.proto for NodeDef and actually
// implemented in core/framework/node_def_util.cc [Note that the proto comment does not include
// dash (-) in names, while the actual implementation permits it. This regex follows the actual
// implementation.]
//
// This pattern is used to ensure fully qualified names always start with a LETTER_DIGIT_DOT,
// followed by zero or more LETTER_DIGIT_DASH_DOT_SLASH_UNDERSCORE. SLASH is not permitted in
// actual user-supplied names to NameScope - it is used as a reserved character to separate
// subcomponents within fully qualified names.
private static final Pattern NAME_REGEX = Pattern.compile("[A-Za-z0-9.][A-Za-z0-9_.\\-]*");
}
tensorflow/java/src/main/java/org/tensorflow/op/Op.java
+35
View File
@@ -0,0 +1,35 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.op;
/**
* A marker interface for all operation wrappers.
*
* <p>Operation wrappers provide strongly typed interfaces for building and execution operations
* without the use of literals and indexes, as required in the core classes.
*
* <p>This interface allows keeping references to any operation wrapper using a common type.
*
* <pre>{@code
* // All values returned by an Ops call can be referred as a Op
* Op split = ops.array().split(...);
* Op shape = ops.array().shape(...);
*
* // All operations could be added to an Op collection
* Collection<Op> allOps = Arrays.asList(split, shape);
* }
*/
public interface Op {}
tensorflow/java/src/main/java/org/tensorflow/op/Operands.java
+46
View File
@@ -0,0 +1,46 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.op;
import java.util.ArrayList;
import java.util.List;
import org.tensorflow.Operand;
import org.tensorflow.OperationBuilder;
import org.tensorflow.Output;
/** Utilities for manipulating operand related types and lists. */
public final class Operands {
/**
* Converts a list of {@link Operand} into an array of {@link Output}.
*
* <p>Operation wrappers need to convert back a list of inputs into an array of outputs in order
* to build an operation, see {@link OperationBuilder#addInputList(Output[])}.
*
* @param inputs an iteration of input operands
* @return an array of outputs
*/
public static Output<?>[] asOutputs(Iterable<? extends Operand<?>> inputs) {
List<Output<?>> outputList = new ArrayList<>();
for (Operand<?> input : inputs) {
outputList.add(input.asOutput());
}
return outputList.toArray(new Output<?>[outputList.size()]);
}
// Disabled constructor
private Operands() {}
}
tensorflow/java/src/main/java/org/tensorflow/op/PrimitiveOp.java
+66
View File
@@ -0,0 +1,66 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.op;
import org.tensorflow.Operation;
/**
* A base class for {@link Op} implementations that are backed by a single {@link Operation}.
*
* <p>Each operation registered in the TensorFlow core is a primitive and is provided as a {@code
* PrimitiveOp}. Custom operations working with only one primitive may also derive from this class.
*/
public abstract class PrimitiveOp implements Op {
/** Returns the underlying {@link Operation} */
public Operation op() {
return operation;
}
@Override
public final int hashCode() {
return operation.hashCode();
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
// Note: we consider that all objects wrapping the same operation are equal, no matter their
// implementation
if (!(obj instanceof PrimitiveOp)) {
return false;
}
return operation.equals(((PrimitiveOp) obj).operation);
}
@Override
public final String toString() {
return String.format("<%s '%s'>", operation.type(), operation.name());
}
protected final Operation operation;
/**
* Constructor.
*
* @param operation the underlying operation
*/
protected PrimitiveOp(Operation operation) {
this.operation = operation;
}
}
tensorflow/java/src/main/java/org/tensorflow/op/Scope.java
+188
View File
@@ -0,0 +1,188 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.op;
import org.tensorflow.ExecutionEnvironment;
import org.tensorflow.Operand;
import org.tensorflow.OperationBuilder;
import java.util.ArrayList;
/**
* Manages groups of related properties when creating Tensorflow Operations, such as a common name
* prefix.
*
* <p>A {@code Scope} is a container for common properties applied to TensorFlow Ops. Normal user
* code initializes a {@code Scope} and provides it to Operation building classes. For example:
*
* <pre>{@code
* Scope scope = new Scope(graph);
* Constant c = Constant.create(scope, 42);
* }</pre>
*
* <p>An Operation building class acquires a Scope, and uses it to set properties on the underlying
* Tensorflow ops. For example:
*
* <pre>{@code
* // An operator class that adds a constant.
* public class Constant {
* public static Constant create(Scope scope, ...) {
* scope.graph().opBuilder(
* "Const", scope.makeOpName("Const"))
* .setAttr(...)
* .build()
* ...
* }
* }
* }</pre>
*
* <p><b>Scope hierarchy:</b>
*
* <p>A {@code Scope} provides various {@code with()} methods that create a new scope. The new scope
* typically has one property changed while other properties are inherited from the parent scope.
*
* <p>An example using {@code Constant} implemented as before:
*
* <pre>{@code
* Scope root = new Scope(graph);
*
* // The linear subscope will generate names like linear/...
* Scope linear = Scope.withSubScope("linear");
*
* // This op name will be "linear/W"
* Constant.create(linear.withName("W"), ...);
*
* // This op will be "linear/Const", using the default
* // name provided by Constant
* Constant.create(linear, ...);
*
* // This op will be "linear/Const_1", using the default
* // name provided by Constant and making it unique within
* // this scope
* Constant.create(linear, ...);
* }</pre>
*
* <p>Scope objects are <b>not</b> thread-safe.
*/
public final class Scope {
/**
* Create a new top-level scope.
*
* @param env The execution environment used by the scope.
*/
public Scope(ExecutionEnvironment env) {
this(env, new NameScope(), new ArrayList<Operand<?>>());
}
/** Returns the execution environment used by this scope. */
public ExecutionEnvironment env() {
return env;
}
/**
* Returns a new scope where added operations will have the provided name prefix.
*
* <p>Ops created with this scope will have {@code name/childScopeName/} as the prefix. The actual
* name will be unique in the returned scope. All other properties are inherited from the current
* scope.
*
* <p>The child scope name must match the regular expression {@code [A-Za-z0-9.][A-Za-z0-9_.\-]*}
*
* @param childScopeName name for the new child scope
* @return a new subscope
* @throws IllegalArgumentException if the name is invalid
*/
public Scope withSubScope(String childScopeName) {
return new Scope(env, nameScope.withSubScope(childScopeName), controlDependencies);
}
/**
* Return a new scope that uses the provided name for an op.
*
* <p>Operations created within this scope will have a name of the form {@code
* name/opName[_suffix]}. This lets you name a specific operator more meaningfully.
*
* <p>Names must match the regular expression {@code [A-Za-z0-9.][A-Za-z0-9_.\-]*}
*
* @param opName name for an operator in the returned scope
* @return a new Scope that uses opName for operations.
* @throws IllegalArgumentException if the name is invalid
*/
public Scope withName(String opName) {
return new Scope(env, nameScope.withName(opName), controlDependencies);
}
/**
* Create a unique name for an operator, using a provided default if necessary.
*
* <p>This is normally called only by operator building classes.
*
* <p>This method generates a unique name, appropriate for the name scope controlled by this
* instance. Typical operator building code might look like
*
* <pre>{@code
* scope.env().opBuilder("Const", scope.makeOpName("Const"))...
* }</pre>
*
* <p><b>Note:</b> if you provide a composite operator building class (i.e, a class that creates a
* set of related operations by calling other operator building code), the provided name will act
* as a subscope to all underlying operators.
*
* @param defaultName name for the underlying operator.
* @return unique name for the operator.
* @throws IllegalArgumentException if the default name is invalid.
*/
public String makeOpName(String defaultName) {
return nameScope.makeOpName(defaultName);
}
private Scope(
ExecutionEnvironment env, NameScope nameScope, Iterable<Operand<?>> controlDependencies) {
this.env = env;
this.nameScope = nameScope;
this.controlDependencies = controlDependencies;
}
/**
* Returns a new scope where added operations will have the provided control dependencies.
*
* <p>Ops created with this scope will have a control edge from each of the provided controls. All
* other properties are inherited from the current scope.
*
* @param controls control dependencies for ops created with the returned scope
* @return a new scope with the provided control dependencies
*/
public Scope withControlDependencies(Iterable<Operand<?>> controls) {
return new Scope(env, nameScope, controls);
}
/**
* Adds each Operand in controlDependencies as a control input to the provided builder.
*
* @param builder OperationBuilder to add control inputs to
*/
public OperationBuilder applyControlDependencies(OperationBuilder builder) {
for (Operand<?> control : controlDependencies) {
builder = builder.addControlInput(control.asOutput().op());
}
return builder;
}
private final ExecutionEnvironment env;
private final Iterable<Operand<?>> controlDependencies;
private final NameScope nameScope;
}
tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java
+112
View File
@@ -0,0 +1,112 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.op.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Annotation used by classes to make TensorFlow operations conveniently accessible via {@code
* org.tensorflow.op.Ops}.
*
* <p>An annotation processor ({@code org.tensorflow.processor.OperatorProcessor}) builds the
* {@code Ops} class by aggregating all classes annotated as {@code @Operator}s. Each annotated
* class <b>must</b> have at least one public static factory method named {@code create} that
* accepts a {@link org.tensorflow.op.Scope} as its first argument. The processor then adds a
* convenience method in the {@code Ops} class. For example:
*
* <pre>{@code
* @Operator
* public final class MyOp implements Op {
* public static MyOp create(Scope scope, Operand operand) {
* ...
* }
* }
* }</pre>
*
* <p>results in a method in the {@code Ops} class
*
* <pre>{@code
* import org.tensorflow.op.Ops;
* ...
* Ops ops = Ops.create(graph);
* ...
* ops.myOp(operand);
* // and has exactly the same effect as calling
* // MyOp.create(ops.getScope(), operand);
* }</pre>
*/
@Documented
@Target(ElementType.TYPE)
@Retention(RetentionPolicy.SOURCE)
public @interface Operator {
/**
* Specify an optional group within the {@code Ops} class.
*
* <p>By default, an annotation processor will create convenience methods directly in the {@code
* Ops} class. An annotated operator may optionally choose to place the method within a group. For
* example:
*
* <pre>{@code
* @Operator(group="math")
* public final class Add extends PrimitiveOp implements Operand {
* ...
* }
* }</pre>
*
* <p>results in the {@code add} method placed within a {@code math} group within the {@code Ops}
* class.
*
* <pre>{@code
* ops.math().add(...);
* }</pre>
*
* <p>The group name must be a <a
* href="https://docs.oracle.com/javase/specs/jls/se7/html/jls-3.html#jls-3.8">valid Java
* identifier</a>.
*/
String group() default "";
/**
* Name for the wrapper method used in the {@code Ops} class.
*
* <p>By default, a processor derives the method name in the {@code Ops} class from the class name
* of the operator. This attribute allow you to provide a different name instead. For example:
*
* <pre>{@code
* @Operator(name="myOperation")
* public final class MyRealOperation implements Operand {
* public static MyRealOperation create(...)
* }
* }</pre>
*
* <p>results in this method added to the {@code Ops} class
*
* <pre>{@code
* ops.myOperation(...);
* // and is the same as calling
* // MyRealOperation.create(...)
* }</pre>
*
* <p>The name must be a <a
* href="https://docs.oracle.com/javase/specs/jls/se7/html/jls-3.html#jls-3.8">valid Java
* identifier</a>.
*/
String name() default "";
}
tensorflow/java/src/main/java/org/tensorflow/op/core/Constant.java
+661
View File
@@ -0,0 +1,661 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.op.core;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.nio.ByteBuffer;
import java.nio.DoubleBuffer;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.nio.LongBuffer;
import java.nio.charset.Charset;
import org.tensorflow.DataType;
import org.tensorflow.Operand;
import org.tensorflow.Operation;
import org.tensorflow.Output;
import org.tensorflow.Tensor;
import org.tensorflow.op.PrimitiveOp;
import org.tensorflow.op.Scope;
import org.tensorflow.op.annotation.Operator;
/** An operator producing a constant value. */
@Operator
public final class Constant<T> extends PrimitiveOp implements Operand<T> {
/**
* Creates a constant containing a single {@code int} element.
*
* @param scope is a scope used to add the underlying operation.
* @param data The value to put into the new constant.
* @return an integer constant
*/
public static Constant<Integer> create(Scope scope, int data) {
return create(scope, data, Integer.class);
}
/**
* Creates a rank-1 constant of {@code int} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Integer> create(Scope scope, int[] data) {
return create(scope, data, Integer.class);
}
/**
* Creates a rank-2 constant of {@code int} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Integer> create(Scope scope, int[][] data) {
return create(scope, data, Integer.class);
}
/**
* Creates a rank-3 constant of {@code int} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Integer> create(Scope scope, int[][][] data) {
return create(scope, data, Integer.class);
}
/**
* Creates a rank-4 constant of {@code int} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Integer> create(Scope scope, int[][][][] data) {
return create(scope, data, Integer.class);
}
/**
* Creates a rank-5 constant of {@code int} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Integer> create(Scope scope, int[][][][][] data) {
return create(scope, data, Integer.class);
}
/**
* Creates a rank-6 constant of {@code int} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Integer> create(Scope scope, int[][][][][][] data) {
return create(scope, data, Integer.class);
}
/**
* Create a {@link DataType#INT32} constant with data from the given buffer.
*
* <p>Creates a constant with the given shape by copying elements from the buffer (starting from
* its current position) into the tensor. For example, if {@code shape = {2,3} } (which represents
* a 2x3 matrix) then the buffer must have 6 elements remaining, which will be consumed by this
* method.
*
* @param scope is a scope used to add the underlying operation.
* @param shape the tensor shape.
* @param data a buffer containing the tensor data.
* @return an integer constant
* @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
*/
public static Constant<Integer> create(Scope scope, long[] shape, IntBuffer data) {
try (Tensor<Integer> value = Tensor.create(shape, data)) {
return createWithTensor(scope, value);
}
}
/**
* Creates a constant containing a single {@code float} element.
*
* @param scope is a scope used to add the underlying operation.
* @param data The value to put into the new constant.
* @return a float constant
*/
public static Constant<Float> create(Scope scope, float data) {
return create(scope, data, Float.class);
}
/**
* Creates a rank-1 constant of {@code float} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Float> create(Scope scope, float[] data) {
return create(scope, data, Float.class);
}
/**
* Creates a rank-2 constant of {@code float} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Float> create(Scope scope, float[][] data) {
return create(scope, data, Float.class);
}
/**
* Creates a rank-3 constant of {@code float} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Float> create(Scope scope, float[][][] data) {
return create(scope, data, Float.class);
}
/**
* Creates a rank-4 constant of {@code float} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Float> create(Scope scope, float[][][][] data) {
return create(scope, data, Float.class);
}
/**
* Creates a rank-5 constant of {@code float} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Float> create(Scope scope, float[][][][][] data) {
return create(scope, data, Float.class);
}
/**
* Creates a rank-6 constant of {@code float} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Float> create(Scope scope, float[][][][][][] data) {
return create(scope, data, Float.class);
}
/**
* Create a {@link DataType#FLOAT} constant with data from the given buffer.
*
* <p>Creates a constant with the given shape by copying elements from the buffer (starting from
* its current position) into the tensor. For example, if {@code shape = {2,3} } (which represents
* a 2x3 matrix) then the buffer must have 6 elements remaining, which will be consumed by this
* method.
*
* @param scope is a scope used to add the underlying operation.
* @param shape the tensor shape.
* @param data a buffer containing the tensor data.
* @return a float constant
* @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
*/
public static Constant<Float> create(Scope scope, long[] shape, FloatBuffer data) {
try (Tensor<Float> value = Tensor.create(shape, data)) {
return createWithTensor(scope, value);
}
}
/**
* Creates a constant containing a single {@code double} element.
*
* @param scope is a scope used to add the underlying operation.
* @param data The value to put into the new constant.
* @return a double constant
*/
public static Constant<Double> create(Scope scope, double data) {
return create(scope, data, Double.class);
}
/**
* Creates a rank-1 constant of {@code double} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Double> create(Scope scope, double[] data) {
return create(scope, data, Double.class);
}
/**
* Creates a rank-2 constant of {@code double} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Double> create(Scope scope, double[][] data) {
return create(scope, data, Double.class);
}
/**
* Creates a rank-3 constant of {@code double} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Double> create(Scope scope, double[][][] data) {
return create(scope, data, Double.class);
}
/**
* Creates a rank-4 constant of {@code double} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Double> create(Scope scope, double[][][][] data) {
return create(scope, data, Double.class);
}
/**
* Creates a rank-5 constant of {@code double} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Double> create(Scope scope, double[][][][][] data) {
return create(scope, data, Double.class);
}
/**
* Creates a rank-6 constant of {@code double} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Double> create(Scope scope, double[][][][][][] data) {
return create(scope, data, Double.class);
}
/**
* Create a {@link DataType#DOUBLE} constant with data from the given buffer.
*
* <p>Creates a constant with the given shape by copying elements from the buffer (starting from
* its current position) into the tensor. For example, if {@code shape = {2,3} } (which represents
* a 2x3 matrix) then the buffer must have 6 elements remaining, which will be consumed by this
* method.
*
* @param scope is a scope used to add the underlying operation.
* @param shape the tensor shape.
* @param data a buffer containing the tensor data.
* @return a double constant
* @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
*/
public static Constant<Double> create(Scope scope, long[] shape, DoubleBuffer data) {
try (Tensor<Double> value = Tensor.create(shape, data)) {
return createWithTensor(scope, value);
}
}
/**
* Creates a constant containing a single {@code long} element.
*
* @param scope is a scope used to add the underlying operation.
* @param data The value to put into the new constant.
* @return a long constant
*/
public static Constant<Long> create(Scope scope, long data) {
return create(scope, data, Long.class);
}
/**
* Creates a rank-1 constant of {@code long} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Long> create(Scope scope, long[] data) {
return create(scope, data, Long.class);
}
/**
* Creates a rank-2 constant of {@code long} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Long> create(Scope scope, long[][] data) {
return create(scope, data, Long.class);
}
/**
* Creates a rank-3 constant of {@code long} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Long> create(Scope scope, long[][][] data) {
return create(scope, data, Long.class);
}
/**
* Creates a rank-4 constant of {@code long} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Long> create(Scope scope, long[][][][] data) {
return create(scope, data, Long.class);
}
/**
* Creates a rank-5 constant of {@code long} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Long> create(Scope scope, long[][][][][] data) {
return create(scope, data, Long.class);
}
/**
* Creates a rank-6 constant of {@code long} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Long> create(Scope scope, long[][][][][][] data) {
return create(scope, data, Long.class);
}
/**
* Create a {@link DataType#INT64} constant with data from the given buffer.
*
* <p>Creates a constant with the given shape by copying elements from the buffer (starting from
* its current position) into the tensor. For example, if {@code shape = {2,3} } (which represents
* a 2x3 matrix) then the buffer must have 6 elements remaining, which will be consumed by this
* method.
*
* @param scope is a scope used to add the underlying operation.
* @param shape the tensor shape.
* @param data a buffer containing the tensor data.
* @return a long constant
* @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
*/
public static Constant<Long> create(Scope scope, long[] shape, LongBuffer data) {
try (Tensor<Long> value = Tensor.create(shape, data)) {
return createWithTensor(scope, value);
}
}
/**
* Creates a constant containing a single {@code boolean} element.
*
* @param scope is a scope used to add the underlying operation.
* @param data The value to put into the new constant.
* @return a boolean constant
*/
public static Constant<Boolean> create(Scope scope, boolean data) {
return create(scope, data, Boolean.class);
}
/**
* Creates a rank-1 constant of {@code boolean} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Boolean> create(Scope scope, boolean[] data) {
return create(scope, data, Boolean.class);
}
/**
* Creates a rank-2 constant of {@code boolean} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Boolean> create(Scope scope, boolean[][] data) {
return create(scope, data, Boolean.class);
}
/**
* Creates a rank-3 constant of {@code boolean} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Boolean> create(Scope scope, boolean[][][] data) {
return create(scope, data, Boolean.class);
}
/**
* Creates a rank-4 constant of {@code boolean} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Boolean> create(Scope scope, boolean[][][][] data) {
return create(scope, data, Boolean.class);
}
/**
* Creates a rank-5 constant of {@code boolean} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Boolean> create(Scope scope, boolean[][][][][] data) {
return create(scope, data, Boolean.class);
}
/**
* Creates a rank-6 constant of {@code boolean} elements.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. The dimensions of the
* new constant will match those of the array.
*/
public static Constant<Boolean> create(Scope scope, boolean[][][][][][] data) {
return create(scope, data, Boolean.class);
}
/**
* Creates a {@code String} constant using the default, UTF-8 encoding.
*
* @param scope is a scope used to add the underlying operation.
* @param data The string to put into the new constant.
* @return a string constant
*/
public static Constant<String> create(Scope scope, String data) {
return create(scope, data, UTF_8);
}
/**
* Creates a {@code String} constant using a specified encoding.
*
* @param scope is a scope used to add the underlying operation.
* @param charset The encoding from String to bytes.
* @param data The string to put into the new constant.
* @return a string constant
*/
public static Constant<String> create(Scope scope, String data, Charset charset) {
try (Tensor<String> value = Tensor.create(data.getBytes(charset), String.class)) {
return createWithTensor(scope, value);
}
}
/**
* Creates a constant containing a single {@code String} element, represented as an array of {@code byte}s.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. String elements are
* sequences of bytes from the last array dimension.
*/
public static Constant<String> create(Scope scope, byte[] data) {
return create(scope, data, String.class);
}
/**
* Creates a rank-1 constant of {@code String} elements, each represented as an array of {@code byte}s.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. String elements are
* sequences of bytes from the last array dimension.
*/
public static Constant<String> create(Scope scope, byte[][] data) {
return create(scope, data, String.class);
}
/**
* Creates a rank-2 constant of {@code String} elements, each represented as an array of {@code byte}s.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. String elements are
* sequences of bytes from the last array dimension.
*/
public static Constant<String> create(Scope scope, byte[][][] data) {
return create(scope, data, String.class);
}
/**
* Creates a rank-3 constant of {@code String} elements, each represented as an array of {@code byte}s.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. String elements are
* sequences of bytes from the last array dimension.
*/
public static Constant<String> create(Scope scope, byte[][][][] data) {
return create(scope, data, String.class);
}
/**
* Creates a rank-4 constant of {@code String} elements, each represented as an array of {@code byte}s.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. String elements are
* sequences of bytes from the last array dimension.
*/
public static Constant<String> create(Scope scope, byte[][][][][] data) {
return create(scope, data, String.class);
}
/**
* Creates a rank-5 constant of {@code String} elements, each represented as an array of {@code byte}s.
*
* @param scope is a scope used to add the underlying operation.
* @param data An array containing the values to put into the new constant. String elements are
* sequences of bytes from the last array dimension.
*/
public static Constant<String> create(Scope scope, byte[][][][][][] data) {
return create(scope, data, String.class);
}
/**
* Create a constant with data from the given buffer.
*
* <p>Creates a Constant with the provided shape of any type where the constant data has been
* encoded into {@code data} as per the specification of the TensorFlow <a
* href="https://www.tensorflow.org/code/tensorflow/c/c_api.h">C
* API</a>.
*
* @param scope is a scope used to add the underlying operation.
* @param type the tensor datatype.
* @param shape the tensor shape.
* @param data a buffer containing the tensor data.
* @return a constant of type `type`
* @throws IllegalArgumentException If the tensor datatype or shape is not compatible with the
* buffer
*/
public static <T> Constant<T> create(Scope scope, Class<T> type, long[] shape, ByteBuffer data) {
try (Tensor<T> value = Tensor.create(type, shape, data)) {
return createWithTensor(scope, value);
}
}
/**
* Create a constant from a Java object.
*
* <p>The argument {@code object} is first converted into a Tensor using {@link
* org.tensorflow.Tensor#create(Object)}, so only Objects supported by this method must be
* provided. For example:
*
* <pre>{@code
* Constant.create(scope, new int[]{{1, 2}, {3, 4}}, Integer.class); // returns a 2x2 integer matrix
* }</pre>
*
* @param scope is a scope used to add the underlying operation.
* @param object a Java object representing the constant.
* @return a constant of type `type`
* @see org.tensorflow.Tensor#create(Object) Tensor.create
*/
public static <T> Constant<T> create(Scope scope, Object object, Class<T> type) {
try (Tensor<T> value = Tensor.create(object, type)) {
return createWithTensor(scope, value);
}
}
private static <T> Constant<T> createWithTensor(Scope scope, Tensor<T> value) {
return new Constant<T>(
scope
.env()
.opBuilder("Const", scope.makeOpName("Const"))
.setAttr("value", value)
.setAttr("dtype", value.dataType())
.build());
}
@Override
public Output<T> asOutput() {
return output;
}
private Constant(Operation operation) {
super(operation);
output = operation.output(0);
}
private final Output<T> output;
}
tensorflow/java/src/main/java/org/tensorflow/op/core/Gradients.java
+168
View File
@@ -0,0 +1,168 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.op.core;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import org.tensorflow.Graph;
import org.tensorflow.Operand;
import org.tensorflow.Output;
import org.tensorflow.op.Op;
import org.tensorflow.op.Operands;
import org.tensorflow.op.Scope;
import org.tensorflow.op.annotation.Operator;
/**
* Adds operations to compute the partial derivatives of sum of {@code y}s w.r.t {@code x}s,
* i.e., {@code d(y_1 + y_2 + ...)/dx_1, d(y_1 + y_2 + ...)/dx_2...}
* <p>
* If {@code Options.dx()} values are set, they are as the initial symbolic partial derivatives of some loss
* function {@code L} w.r.t. {@code y}. {@code Options.dx()} must have the size of {@code y}.
* <p>
* If {@code Options.dx()} is not set, the implementation will use dx of {@code OnesLike} for all
* shapes in {@code y}.
* <p>
* The partial derivatives are returned in output {@code dy}, with the size of {@code x}.
* <p>
* Example of usage:
* <pre>{@code
* Gradients gradients = Gradients.create(scope, Arrays.asList(loss), Arrays.asList(w, b));
*
* Constant<Float> alpha = ops.constant(1.0f, Float.class);
* ApplyGradientDescent.create(scope, w, alpha, gradients.<Float>dy(0));
* ApplyGradientDescent.create(scope, b, alpha, gradients.<Float>dy(1));
* }</pre>
*/
@Operator
public class Gradients implements Op, Iterable<Operand<?>> {
/**
* Optional attributes for {@link Gradients}
*/
public static class Options {
/**
* @param dx partial derivatives of some loss function {@code L} w.r.t. {@code y}
* @return this option builder
*/
public Options dx(Iterable<? extends Operand<?>> dx) {
this.dx = dx;
return this;
}
private Iterable<? extends Operand<?>> dx;
private Options() {
}
}
/**
* Adds gradients computation ops to the graph according to scope.
*
* @param scope current graph scope
* @param y outputs of the function to derive
* @param x inputs of the function for which partial derivatives are computed
* @param options carries optional attributes values
* @return a new instance of {@code Gradients}
* @throws IllegalArgumentException if execution environment is not a graph
*/
public static Gradients create(
Scope scope,
Iterable<? extends Operand<?>> y,
Iterable<? extends Operand<?>> x,
Options... options) {
if (!(scope.env() instanceof Graph)) {
throw new IllegalArgumentException(
"Gradients can be computed only in a graph execution environment");
}
Graph graph = (Graph) scope.env();
Output<?>[] dx = null;
if (options != null) {
for (Options opts : options) {
if (opts.dx != null) {
dx = Operands.asOutputs(opts.dx);
}
}
}
Output<?>[] dy =
graph.addGradients(
scope.makeOpName("Gradients"), Operands.asOutputs(y), Operands.asOutputs(x), dx);
return new Gradients(Arrays.asList(dy));
}
/**
* Adds gradients computation ops to the graph according to scope.
*
* <p>This is a simplified version of {@link #create(Scope, Iterable, Iterable, Options...)} where
* {@code y} is a single output.
*
* @param scope current graph scope
* @param y output of the function to derive
* @param x inputs of the function for which partial derivatives are computed
* @param options carries optional attributes values
* @return a new instance of {@code Gradients}
* @throws IllegalArgumentException if execution environment is not a graph
*/
@SuppressWarnings({"unchecked", "rawtypes"})
public static Gradients create(
Scope scope, Operand<?> y, Iterable<? extends Operand<?>> x, Options... options) {
return create(scope, (Iterable) Arrays.asList(y), x, options);
}
/**
* @param dx partial derivatives of some loss function {@code L} w.r.t. {@code y}
* @return builder to add more options to this operation
*/
public static Options dx(Iterable<? extends Operand<?>> dx) {
return new Options().dx(dx);
}
@Override
@SuppressWarnings({"rawtypes", "unchecked"})
public Iterator<Operand<?>> iterator() {
return (Iterator) dy.iterator();
}
/**
* Partial derivatives of {@code y}s w.r.t. {@code x}s, with the size of {@code x}
*/
public List<Output<?>> dy() {
return dy;
}
/**
* Returns a symbolic handle to one of the gradient operation output
*
* <p>Warning: Does not check that the type of the tensor matches T. It is recommended to call
* this method with an explicit type parameter rather than letting it be inferred, e.g. {@code
* gradients.<Float>dy(0)}
*
* @param <T> The expected element type of the tensors produced by this output.
* @param index The index of the output among the gradients added by this operation
*/
@SuppressWarnings("unchecked")
public <T> Output<T> dy(int index) {
return (Output<T>) dy.get(index);
}
private List<Output<?>> dy;
private Gradients(List<Output<?>> dy) {
this.dy = dy;
}
}
tensorflow/java/src/main/java/org/tensorflow/op/core/Zeros.java
+68
View File
@@ -0,0 +1,68 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.op.core;
import java.nio.ByteBuffer;
import org.tensorflow.DataType;
import org.tensorflow.Operand;
import org.tensorflow.Output;
import org.tensorflow.op.Op;
import org.tensorflow.op.Scope;
import org.tensorflow.op.annotation.Operator;
/**
* An operator creating a constant initialized with zeros of the shape given by `dims`.
*
* <p>For example, the following expression
* <pre>{@code ops.zeros(ops.constant(new long[]{2, 2}), Float.class)}</pre>
* is the equivalent of
* <pre>{@code ops.fill(ops.constant(new long[]{2, 2}), ops.constant(0.0f))}</pre>
*
* @param <T> constant type
*/
@Operator
public class Zeros<T> implements Op, Operand<T> {
/**
* Creates a zeroed tensor given its type and shape.
*
* @param scope is a scope used to add the underlying operation
* @param dims a 1-D operand that represents the shape of the output tensor
* @param type the output tensor datatype
* @return a constant tensor initialized with zeros
* @throws IllegalArgumentException if the tensor type or shape cannot be initialized with zeros.
*/
public static <T, U extends Number> Zeros<T> create(Scope scope, Operand<U> dims, Class<T> type) {
Scope childScope = scope.withSubScope("Zeros"); // If scope had an op name set, it will prevail on "Zeros"
int zeroSize = DataType.fromClass(type).byteSize();
if (zeroSize < 0) {
throw new IllegalArgumentException(type.getSimpleName() + " tensors cannot be initialized with zeros");
}
Constant<T> zero = Constant.create(childScope.withName("Zero"), type, new long[]{}, ByteBuffer.allocate(zeroSize));
return new Zeros<T>(Fill.create(childScope, dims, zero));
}
@Override
public Output<T> asOutput() {
return fill.asOutput();
}
private final Fill<T> fill;
private Zeros(Fill<T> fill) {
this.fill = fill;
}
}
tensorflow/java/src/main/java/org/tensorflow/package-info.java
+45
View File
@@ -0,0 +1,45 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
/**
* Defines classes to build, save, load and execute TensorFlow models.
*
*<aside class="warning">
* <b>Warning:</b> This API is deprecated and will be removed in a future
* version of TensorFlow after <a href="https://tensorflow.org/java">the replacement</a>
* is stable.
*</aside>
*
* <p>To get started, see the <a href="https://tensorflow.org/install/lang_java_legacy">
* installation instructions.</a></p>
*
* <p>The <a
* href="https://www.tensorflow.org/code/tensorflow/java/src/main/java/org/tensorflow/examples/LabelImage.java">LabelImage</a>
* example demonstrates use of this API to classify images using a pre-trained <a
* href="http://arxiv.org/abs/1512.00567">Inception</a> architecture convolutional neural network.
* It demonstrates:
*
* <ul>
* <li>Graph construction: using the OperationBuilder class to construct a graph to decode, resize
* and normalize a JPEG image.
* <li>Model loading: Using Graph.importGraphDef() to load a pre-trained Inception model.
* <li>Graph execution: Using a Session to execute the graphs and find the best label for an
* image.
* </ul>
*
* <p>Additional examples can be found in the <a
* href="https://github.com/tensorflow/java">tensorflow/java</a> GitHub repository.
*/
package org.tensorflow;
tensorflow/java/src/main/java/org/tensorflow/types/UInt8.java
+48
View File
@@ -0,0 +1,48 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.types;
/** Represents an 8-bit unsigned integer. */
public class UInt8 extends Number {
private static final long serialVersionUID = 1L;
// This class is only used for generic parameterization and is not instantiable. Thus,
// it is safe to implement the Number abstract methods with all zeros, as they will
// never be invoked.
@Override
public double doubleValue() {
return 0.0;
}
@Override
public float floatValue() {
return 0.0f;
}
@Override
public int intValue() {
return 0;
}
@Override
public long longValue() {
return 0L;
}
private UInt8() {}
}
tensorflow/java/src/main/java/org/tensorflow/types/package-info.java
+29
View File
@@ -0,0 +1,29 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
/**
* Defines classes that represent TensorFlow data types. For each possible data type that can be
* used in a tensor, there is a corresponding class that is used to represent it. For example, the
* TensorFlow int32 type is represented by the type {@link java.lang.Integer} and by the class
* object {@code Integer.class}. The former is used to support compile-time checking of tensor
* element types and the latter is used for run-time checking of element types. Classes appearing in
* this package, such as UInt8, represent TensorFlow data types for which there is no existing Java
* equivalent.
*
* <p>TensorFlow element types are also separately represented by the {@link
* org.tensorflow.DataType} enum, with one enum value per element type. The enum representation is
* not usually needed, but can be obtained using {@link org.tensorflow.DataType#fromClass}.
*/
package org.tensorflow.types;
tensorflow/java/src/main/native/BUILD
+53
View File
@@ -0,0 +1,53 @@
# Description:
# Java Native Interface (JNI) library intended for implementing the
# TensorFlow Java API using the TensorFlow C library.
load("//tensorflow:tensorflow.bzl", "tf_copts", "tf_cuda_library")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = [
"//tensorflow/java:__pkg__",
# TODO(ashankar): Temporary hack for the Java API and
# //third_party/tensorflow/tools/android/inference_interface:android_tensorflow_inference_jni
# to co-exist in a single shared library. However, the hope is that
# //third_party/tensorflow/tools/android/inference_interface:android_tensorflow_jni can be
# removed once the Java API provides feature parity with it.
"//tensorflow/tools/android/inference_interface:__pkg__",
],
licenses = ["notice"],
)
filegroup(
name = "native_srcs",
srcs = glob([
"*.cc",
"*.h",
]),
visibility = ["//visibility:public"],
)
tf_cuda_library(
name = "native",
srcs = glob(["*.cc"]),
hdrs = glob(["*.h"]),
copts = tf_copts(),
features = select({
"//tensorflow:android": ["-layering_check"],
"//conditions:default": [],
}),
deps = select({
"//tensorflow:android": [
"//tensorflow/core:portable_tensorflow_lib",
],
"//conditions:default": [
"//tensorflow/c:c_api",
"//tensorflow/c/eager:c_api",
"//tensorflow/core:all_kernels",
"//tensorflow/core:direct_session",
"//tensorflow/core:ops",
"@bazel_tools//tools/jdk:jni",
],
}),
alwayslink = 1,
)
tensorflow/java/src/main/native/eager_operation_builder_jni.cc
+335
View File
@@ -0,0 +1,335 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/java/src/main/native/eager_operation_builder_jni.h"
#include <cstdint>
#include <cstring>
#include <memory>
#include "tensorflow/c/eager/c_api.h"
#include "tensorflow/java/src/main/native/exception_jni.h"
// This value should be >= to the maximum number of outputs in any op
#define MAX_OUTPUTS_PER_OP 8
namespace {
TFE_Op* requireOp(JNIEnv* env, jlong handle) {
if (handle == 0) {
throwException(env, kIllegalStateException,
"Operation has already been built");
return nullptr;
}
return reinterpret_cast<TFE_Op*>(handle);
}
TFE_Context* requireContext(JNIEnv* env, jlong handle) {
if (handle == 0) {
throwException(env, kIllegalStateException, "Context has been deleted");
return nullptr;
}
return reinterpret_cast<TFE_Context*>(handle);
}
TF_Tensor* requireTensor(JNIEnv* env, jlong handle) {
if (handle == 0) {
throwException(env, kIllegalStateException,
"close() has been called on the Tensor");
return nullptr;
}
return reinterpret_cast<TF_Tensor*>(handle);
}
TFE_TensorHandle* requireTensorHandle(JNIEnv* env, jlong handle) {
if (handle == 0) {
throwException(env, kIllegalStateException,
"Tensor handle has been deleted");
return nullptr;
}
return reinterpret_cast<TFE_TensorHandle*>(handle);
}
} // namespace
JNIEXPORT jlong JNICALL Java_org_tensorflow_EagerOperationBuilder_allocate(
JNIEnv* env, jclass clazz, jlong context_handle, jstring name) {
TFE_Context* context = requireContext(env, context_handle);
if (context == nullptr) return 0;
const char* op_or_function_name = env->GetStringUTFChars(name, nullptr);
TF_Status* status = TF_NewStatus();
TFE_Op* op = TFE_NewOp(context, op_or_function_name, status);
env->ReleaseStringUTFChars(name, op_or_function_name);
if (!throwExceptionIfNotOK(env, status)) {
TF_DeleteStatus(status);
return 0;
}
TF_DeleteStatus(status);
static_assert(sizeof(jlong) >= sizeof(TFE_Op*),
"Cannot represent a C TFE_Op as a Java long");
return reinterpret_cast<jlong>(op);
}
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_delete(
JNIEnv* env, jclass clazz, jlong op_handle) {
if (op_handle == 0) return;
TFE_DeleteOp(reinterpret_cast<TFE_Op*>(op_handle));
}
JNIEXPORT jlongArray JNICALL Java_org_tensorflow_EagerOperationBuilder_execute(
JNIEnv* env, jclass clazz, jlong op_handle) {
TFE_Op* op = requireOp(env, op_handle);
if (op == nullptr) return nullptr;
int num_retvals = MAX_OUTPUTS_PER_OP;
std::unique_ptr<TFE_TensorHandle*[]> retvals(
new TFE_TensorHandle*[num_retvals]);
TF_Status* status = TF_NewStatus();
TFE_Execute(op, retvals.get(), &num_retvals, status);
if (!throwExceptionIfNotOK(env, status)) {
TF_DeleteStatus(status);
return nullptr;
}
TF_DeleteStatus(status);
jlongArray rethandles = env->NewLongArray(num_retvals);
if (num_retvals > 0) {
jlong* retval = env->GetLongArrayElements(rethandles, nullptr);
for (int i = 0; i < num_retvals; ++i) {
retval[i] = reinterpret_cast<jlong>(retvals[i]);
}
env->ReleaseLongArrayElements(rethandles, retval, 0);
}
return rethandles;
}
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_setDevice(
JNIEnv* env, jclass clazz, jlong op_handle, jstring device_name) {
TFE_Op* op = requireOp(env, op_handle);
if (op == nullptr) return;
const char* cname = env->GetStringUTFChars(device_name, nullptr);
TF_Status* status = TF_NewStatus();
TFE_OpSetDevice(op, cname, status);
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
env->ReleaseStringUTFChars(device_name, cname);
}
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_addInput(
JNIEnv* env, jclass clazz, jlong op_handle, jlong input_handle) {
TFE_Op* op = requireOp(env, op_handle);
if (op == nullptr) return;
TFE_TensorHandle* tensor_handle = requireTensorHandle(env, input_handle);
if (tensor_handle == nullptr) return;
TF_Status* status = TF_NewStatus();
TFE_OpAddInput(op, tensor_handle, status);
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
}
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_addInputList(
JNIEnv* env, jclass clazz, jlong op_handle, jlongArray input_handles) {
TFE_Op* op = requireOp(env, op_handle);
if (op == nullptr) return;
jlong* cinput_handles = env->GetLongArrayElements(input_handles, nullptr);
size_t num_inputs = static_cast<size_t>(env->GetArrayLength(input_handles));
std::unique_ptr<TFE_TensorHandle*[]> tensor_handles(
new TFE_TensorHandle*[num_inputs]);
for (int i = 0; i < num_inputs; ++i) {
tensor_handles[i] = requireTensorHandle(env, cinput_handles[i]);
if (tensor_handles[i] == nullptr) {
env->ReleaseLongArrayElements(input_handles, cinput_handles, JNI_ABORT);
return;
}
}
env->ReleaseLongArrayElements(input_handles, cinput_handles, JNI_ABORT);
TF_Status* status = TF_NewStatus();
TFE_OpAddInputList(op, tensor_handles.get(), num_inputs, status);
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
}
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_setAttrString(
JNIEnv* env, jclass clazz, jlong op_handle, jstring attr_name,
jbyteArray value) {
static_assert(sizeof(jbyte) == 1,
"Require Java byte to be represented as a single byte");
TFE_Op* op = requireOp(env, op_handle);
if (op == nullptr) return;
const char* cname = env->GetStringUTFChars(attr_name, nullptr);
jbyte* cvalue = env->GetByteArrayElements(value, nullptr);
TFE_OpSetAttrString(op, cname, cvalue, env->GetArrayLength(value));
env->ReleaseByteArrayElements(value, cvalue, JNI_ABORT);
env->ReleaseStringUTFChars(attr_name, cname);
}
JNIEXPORT void JNICALL
Java_org_tensorflow_EagerOperationBuilder_setAttrStringList(
JNIEnv* env, jclass object, jlong op_handle, jstring attr_name,
jobjectArray values) {
TFE_Op* op = requireOp(env, op_handle);
if (op == nullptr) return;
const char* cname = env->GetStringUTFChars(attr_name, nullptr);
int num_values = env->GetArrayLength(values);
static_assert(sizeof(jbyte) == 1,
"Require Java byte to be represented as a single byte");
std::unique_ptr<jbyteArray[]> jarrays(new jbyteArray[num_values]);
std::unique_ptr<jbyte*[]> jvalues(new jbyte*[num_values]);
std::unique_ptr<void*[]> cvalues(new void*[num_values]);
std::unique_ptr<size_t[]> lengths(new size_t[num_values]);
for (int i = 0; i < num_values; ++i) {
jbyteArray v =
static_cast<jbyteArray>(env->GetObjectArrayElement(values, i));
jarrays[i] = v;
jvalues[i] = env->GetByteArrayElements(v, nullptr);
cvalues[i] = jvalues[i];
lengths[i] = static_cast<size_t>(env->GetArrayLength(v));
}
TFE_OpSetAttrStringList(op, cname, cvalues.get(), lengths.get(), num_values);
for (int i = 0; i < num_values; ++i) {
env->ReleaseByteArrayElements(jarrays[i], jvalues[i], JNI_ABORT);
}
env->ReleaseStringUTFChars(attr_name, cname);
}
#define DEFINE_SET_ATTR_SCALAR(name, jtype, ctype) \
JNIEXPORT void JNICALL \
Java_org_tensorflow_EagerOperationBuilder_setAttr##name( \
JNIEnv* env, jclass clazz, jlong op_handle, jstring attr_name, \
jtype value) { \
static_assert( \
sizeof(ctype) >= sizeof(jtype), \
"Information loss when converting between Java and C types"); \
TFE_Op* op = requireOp(env, op_handle); \
if (op == nullptr) return; \
const char* cname = env->GetStringUTFChars(attr_name, nullptr); \
TFE_OpSetAttr##name(op, cname, static_cast<ctype>(value)); \
env->ReleaseStringUTFChars(attr_name, cname); \
}
#define DEFINE_SET_ATTR_LIST(name, jname, jtype, ctype) \
JNIEXPORT void JNICALL \
Java_org_tensorflow_EagerOperationBuilder_setAttr##name##List( \
JNIEnv* env, jclass clazz, jlong op_handle, jstring attr_name, \
jtype##Array value) { \
TFE_Op* op = requireOp(env, op_handle); \
if (op == nullptr) return; \
const char* cname = env->GetStringUTFChars(attr_name, nullptr); \
/* Make a copy of the array to paper over any differences */ \
/* in byte representations of the jtype and ctype */ \
/* For example, jint vs TF_DataType. */ \
/* If this copy turns out to be a problem in practice */ \
/* can avoid it for many types. */ \
const int n = env->GetArrayLength(value); \
std::unique_ptr<ctype[]> cvalue(new ctype[n]); \
jtype* elems = env->Get##jname##ArrayElements(value, nullptr); \
for (int i = 0; i < n; ++i) { \
cvalue[i] = static_cast<ctype>(elems[i]); \
} \
TFE_OpSetAttr##name##List(op, cname, cvalue.get(), n); \
env->Release##jname##ArrayElements(value, elems, JNI_ABORT); \
env->ReleaseStringUTFChars(attr_name, cname); \
}
#define DEFINE_SET_ATTR(name, jname, jtype, ctype) \
DEFINE_SET_ATTR_SCALAR(name, jtype, ctype) \
DEFINE_SET_ATTR_LIST(name, jname, jtype, ctype)
DEFINE_SET_ATTR(Int, Long, jlong, int64_t);
DEFINE_SET_ATTR(Float, Float, jfloat, float);
DEFINE_SET_ATTR(Bool, Boolean, jboolean, unsigned char);
DEFINE_SET_ATTR(Type, Int, jint, TF_DataType);
#undef DEFINE_SET_ATTR
#undef DEFINE_SET_ATTR_LIST
#undef DEFINE_SET_ATTR_SCALAR
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_setAttrTensor(
JNIEnv* env, jclass clazz, jlong handle, jstring attr_name,
jlong tensor_handle) {
TFE_Op* op = requireOp(env, handle);
if (op == nullptr) return;
TF_Tensor* t = requireTensor(env, tensor_handle);
if (t == nullptr) return;
const char* cname = env->GetStringUTFChars(attr_name, nullptr);
TF_Status* status = TF_NewStatus();
TFE_OpSetAttrTensor(op, cname, t, status);
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
env->ReleaseStringUTFChars(attr_name, cname);
}
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_setAttrShape(
JNIEnv* env, jclass clazz, jlong op_handle, jstring attr_name,
jlongArray shape, jint num_dims) {
TFE_Op* op = requireOp(env, op_handle);
if (op == nullptr) return;
std::unique_ptr<int64_t[]> cvalue;
// num_dims and env->GetArrayLength(shape) are assumed to be consistent.
// i.e., either num_dims < 0 or num_dims == env->GetArrayLength(shape).
if (num_dims > 0) {
cvalue.reset(new int64_t[num_dims]);
jlong* elems = env->GetLongArrayElements(shape, nullptr);
for (int i = 0; i < num_dims; ++i) {
cvalue[i] = static_cast<int64_t>(elems[i]);
}
env->ReleaseLongArrayElements(shape, elems, JNI_ABORT);
}
const char* cname = env->GetStringUTFChars(attr_name, nullptr);
TF_Status* status = TF_NewStatus();
TFE_OpSetAttrShape(op, cname, cvalue.get(), static_cast<int>(num_dims),
status);
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
env->ReleaseStringUTFChars(attr_name, cname);
}
JNIEXPORT void JNICALL
Java_org_tensorflow_EagerOperationBuilder_setAttrShapeList(
JNIEnv* env, jclass clazz, jlong op_handle, jstring attr_name,
jlongArray shapes, jintArray num_dims) {
TFE_Op* op = requireOp(env, op_handle);
if (op == nullptr) return;
std::unique_ptr<int64_t[]> cshapes;
std::unique_ptr<const int64_t*[]> cdims;
std::unique_ptr<int[]> cnum_dims;
const int num_dims_length = env->GetArrayLength(num_dims);
if (num_dims_length > 0) {
const int shapes_length = env->GetArrayLength(shapes);
cshapes.reset(new int64_t[shapes_length]);
cdims.reset(new const int64_t*[num_dims_length]);
cnum_dims.reset(new int[num_dims_length]);
jlong* shapes_elems =
static_cast<jlong*>(env->GetPrimitiveArrayCritical(shapes, nullptr));
std::memcpy(cshapes.get(), shapes_elems, shapes_length << 3);
env->ReleasePrimitiveArrayCritical(shapes, shapes_elems, JNI_ABORT);
int64_t* cshapes_ptr = cshapes.get();
jint* num_dims_elems =
static_cast<jint*>(env->GetPrimitiveArrayCritical(num_dims, nullptr));
for (int i = 0; i < num_dims_length; ++i) {
cnum_dims[i] = static_cast<int>(num_dims_elems[i]);
cdims[i] = cshapes_ptr;
if (cnum_dims[i] > 0) {
cshapes_ptr += cnum_dims[i];
}
}
env->ReleasePrimitiveArrayCritical(num_dims, num_dims_elems, JNI_ABORT);
}
const char* cname = env->GetStringUTFChars(attr_name, nullptr);
TF_Status* status = TF_NewStatus();
TFE_OpSetAttrShapeList(op, cname, cdims.get(), cnum_dims.get(),
num_dims_length, status);
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
env->ReleaseStringUTFChars(attr_name, cname);
}
tensorflow/java/src/main/native/eager_operation_builder_jni.h
+191
View File
@@ -0,0 +1,191 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_MAIN_NATIVE_EAGER_OPERATION_BUILDER_JNI_H_
#define TENSORFLOW_JAVA_SRC_MAIN_NATIVE_EAGER_OPERATION_BUILDER_JNI_H_
#include <jni.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: allocate
* Signature: (JLjava/lang/String;)J
*/
JNIEXPORT jlong JNICALL Java_org_tensorflow_EagerOperationBuilder_allocate(
JNIEnv *, jclass, jlong, jstring);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: delete
* Signature: (J)V
*/
JNIEXPORT void JNICALL
Java_org_tensorflow_EagerOperationBuilder_delete(JNIEnv *, jclass, jlong);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: execute
* Signature: (J)[J
*/
JNIEXPORT jlongArray JNICALL
Java_org_tensorflow_EagerOperationBuilder_execute(JNIEnv *, jclass, jlong);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: addInput
* Signature: (JJ)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_addInput(
JNIEnv *, jclass, jlong, jlong);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: addInputList
* Signature: (J[J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_addInputList(
JNIEnv *, jclass, jlong, jlongArray);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: setDevice
* Signature: (JLjava/lang/String;)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_setDevice(
JNIEnv *, jclass, jlong, jstring);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: setAttrString
* Signature: (JLjava/lang/String;[B)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_setAttrString(
JNIEnv *, jclass, jlong, jstring, jbyteArray);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: setAttrStringList
* Signature: (JLjava/lang/String;[L)V
*/
JNIEXPORT void JNICALL
Java_org_tensorflow_EagerOperationBuilder_setAttrStringList(JNIEnv *, jclass,
jlong, jstring,
jobjectArray);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: setAttrInt
* Signature: (JLjava/lang/String;J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_setAttrInt(
JNIEnv *, jclass, jlong, jstring, jlong);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: setAttrIntList
* Signature: (JLjava/lang/String;[J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_setAttrIntList(
JNIEnv *, jclass, jlong, jstring, jlongArray);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: setAttrFloat
* Signature: (JLjava/lang/String;F)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_setAttrFloat(
JNIEnv *, jclass, jlong, jstring, jfloat);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: setAttrFloatList
* Signature: (JLjava/lang/String;[F)V
*/
JNIEXPORT void JNICALL
Java_org_tensorflow_EagerOperationBuilder_setAttrFloatList(JNIEnv *, jclass,
jlong, jstring,
jfloatArray);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: setAttrBool
* Signature: (JLjava/lang/String;Z)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_setAttrBool(
JNIEnv *, jclass, jlong, jstring, jboolean);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: setAttrBoolList
* Signature: (JLjava/lang/String;[Z)V
*/
JNIEXPORT void JNICALL
Java_org_tensorflow_EagerOperationBuilder_setAttrBoolList(JNIEnv *, jclass,
jlong, jstring,
jbooleanArray);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: setAttrType
* Signature: (JLjava/lang/String;I)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_setAttrType(
JNIEnv *, jclass, jlong, jstring, jint);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: setAttrTypeList
* Signature: (JLjava/lang/String;[I)V
*/
JNIEXPORT void JNICALL
Java_org_tensorflow_EagerOperationBuilder_setAttrTypeList(JNIEnv *, jclass,
jlong, jstring,
jintArray);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: setAttrTensor
* Signature: (JLjava/lang/String;J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_setAttrTensor(
JNIEnv *, jclass, jlong, jstring, jlong);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: setAttrShape
* Signature: (JLjava/lang/String;[JI)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperationBuilder_setAttrShape(
JNIEnv *, jclass, jlong, jstring, jlongArray, jint);
/*
* Class: org_tensorflow_EagerOperationBuilder
* Method: setAttrShapeList
* Signature: (JLjava/lang/String;[J[I)V
*/
JNIEXPORT void JNICALL
Java_org_tensorflow_EagerOperationBuilder_setAttrShapeList(JNIEnv *, jclass,
jlong, jstring,
jlongArray,
jintArray);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif // TENSORFLOW_JAVA_SRC_MAIN_NATIVE_EAGER_OPERATION_BUILDER_JNI_H_
tensorflow/java/src/main/native/eager_operation_jni.cc
+144
View File
@@ -0,0 +1,144 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/java/src/main/native/eager_operation_jni.h"
#include <assert.h>
#include <stdlib.h>
#include <cstdint>
#include "tensorflow/c/eager/c_api.h"
#include "tensorflow/java/src/main/native/exception_jni.h"
namespace {
TFE_Op* requireOp(JNIEnv* env, jlong handle) {
if (handle == 0) {
throwException(env, kIllegalStateException,
"Eager session has been closed");
return nullptr;
}
return reinterpret_cast<TFE_Op*>(handle);
}
TFE_TensorHandle* requireTensorHandle(JNIEnv* env, jlong handle) {
if (handle == 0) {
throwException(env, kIllegalStateException, "EagerSession has been closed");
return nullptr;
}
return reinterpret_cast<TFE_TensorHandle*>(handle);
}
} // namespace
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperation_delete(JNIEnv* env,
jclass clazz,
jlong handle) {
if (handle == 0) return;
TFE_DeleteOp(reinterpret_cast<TFE_Op*>(handle));
}
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperation_deleteTensorHandle(
JNIEnv* env, jclass clazz, jlong handle) {
if (handle == 0) return;
TFE_DeleteTensorHandle(reinterpret_cast<TFE_TensorHandle*>(handle));
}
JNIEXPORT jlong JNICALL Java_org_tensorflow_EagerOperation_resolveTensorHandle(
JNIEnv* env, jclass clazz, jlong handle) {
TFE_TensorHandle* tensor_handle = requireTensorHandle(env, handle);
if (tensor_handle == nullptr) return 0;
TF_Status* status = TF_NewStatus();
TF_Tensor* tensor = TFE_TensorHandleResolve(tensor_handle, status);
if (!throwExceptionIfNotOK(env, status)) {
TF_DeleteStatus(status);
return 0;
}
TF_DeleteStatus(status);
static_assert(sizeof(jlong) >= sizeof(TF_Tensor*),
"Cannot represent a C TF_Tensor as a Java long");
return reinterpret_cast<jlong>(tensor);
}
JNIEXPORT jint JNICALL Java_org_tensorflow_EagerOperation_outputListLength(
JNIEnv* env, jclass clazz, jlong handle, jstring name) {
TFE_Op* op = requireOp(env, handle);
if (op == nullptr) return 0;
TF_Status* status = TF_NewStatus();
const char* cname = env->GetStringUTFChars(name, nullptr);
int length = TFE_OpGetOutputLength(op, cname, status);
env->ReleaseStringUTFChars(name, cname);
if (!throwExceptionIfNotOK(env, status)) {
TF_DeleteStatus(status);
return 0;
}
TF_DeleteStatus(status);
return static_cast<jint>(length);
}
JNIEXPORT jint JNICALL Java_org_tensorflow_EagerOperation_inputListLength(
JNIEnv* env, jclass clazz, jlong handle, jstring name) {
TFE_Op* op = requireOp(env, handle);
if (op == nullptr) return 0;
TF_Status* status = TF_NewStatus();
const char* cname = env->GetStringUTFChars(name, nullptr);
int length = TFE_OpGetInputLength(op, cname, status);
env->ReleaseStringUTFChars(name, cname);
if (!throwExceptionIfNotOK(env, status)) {
TF_DeleteStatus(status);
return 0;
}
TF_DeleteStatus(status);
return static_cast<jint>(length);
}
JNIEXPORT jint JNICALL Java_org_tensorflow_EagerOperation_dataType(
JNIEnv* env, jclass clazz, jlong handle) {
TFE_TensorHandle* tensor_handle = requireTensorHandle(env, handle);
if (tensor_handle == nullptr) return 0;
TF_DataType data_type = TFE_TensorHandleDataType(tensor_handle);
return static_cast<jint>(data_type);
}
JNIEXPORT jint JNICALL Java_org_tensorflow_EagerOperation_numDims(
JNIEnv* env, jclass clazz, jlong handle) {
TFE_TensorHandle* tensor_handle = requireTensorHandle(env, handle);
if (tensor_handle == nullptr) return 0;
TF_Status* status = TF_NewStatus();
int num_dims = TFE_TensorHandleNumDims(tensor_handle, status);
if (!throwExceptionIfNotOK(env, status)) {
TF_DeleteStatus(status);
return 0;
}
TF_DeleteStatus(status);
return static_cast<jint>(num_dims);
}
JNIEXPORT jlong JNICALL Java_org_tensorflow_EagerOperation_dim(JNIEnv* env,
jclass clazz,
jlong handle,
jint dim_index) {
TFE_TensorHandle* tensor_handle = requireTensorHandle(env, handle);
if (tensor_handle == nullptr) return 0;
TF_Status* status = TF_NewStatus();
int64_t dim = TFE_TensorHandleDim(tensor_handle, dim_index, status);
if (!throwExceptionIfNotOK(env, status)) {
TF_DeleteStatus(status);
return 0;
}
TF_DeleteStatus(status);
return static_cast<jlong>(dim);
}
tensorflow/java/src/main/native/eager_operation_jni.h
+94
View File
@@ -0,0 +1,94 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_MAIN_NATIVE_EAGER_OPERATION_JNI_H_
#define TENSORFLOW_JAVA_SRC_MAIN_NATIVE_EAGER_OPERATION_JNI_H_
#include <jni.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
* Class: org_tensorflow_EagerOperation
* Method: delete
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_EagerOperation_delete(JNIEnv *,
jclass, jlong);
/*
* Class: org_tensorflow_EagerOperation
* Method: deleteTensorHandle
* Signature: (J)V
*/
JNIEXPORT void JNICALL
Java_org_tensorflow_EagerOperation_deleteTensorHandle(JNIEnv *, jclass, jlong);
/**
* Class: org_tensorflow_EagerOperation
* Method: resolveTensorHandle
* Signature: (J)J
*/
JNIEXPORT jlong JNICALL
Java_org_tensorflow_EagerOperation_resolveTensorHandle(JNIEnv *, jclass, jlong);
/**
* Class: org_tensorflow_EagerOperation
* Method: outputListLength
* Signature: (JLjava/lang/String;)I
*/
JNIEXPORT jint JNICALL Java_org_tensorflow_EagerOperation_outputListLength(
JNIEnv *, jclass, jlong, jstring);
/**
* Class: org_tensorflow_EagerOperation
* Method: inputListLength
* Signature: (JLjava/lang/String;)I
*/
JNIEXPORT jint JNICALL Java_org_tensorflow_EagerOperation_inputListLength(
JNIEnv *, jclass, jlong, jstring);
/**
* Class: org_tensorflow_EagerOperation
* Method: dataType
* Signature: (J)I
*/
JNIEXPORT jint JNICALL Java_org_tensorflow_EagerOperation_dataType(JNIEnv *,
jclass,
jlong);
/**
* Class: org_tensorflow_EagerOperation
* Method: numDims
* Signature: (J)I
*/
JNIEXPORT jint JNICALL Java_org_tensorflow_EagerOperation_numDims(JNIEnv *,
jclass,
jlong);
/**
* Class: org_tensorflow_EagerOperation
* Method: dim
* Signature: (JI)J
*/
JNIEXPORT jlong JNICALL Java_org_tensorflow_EagerOperation_dim(JNIEnv *, jclass,
jlong, jint);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif // TENSORFLOW_JAVA_SRC_MAIN_NATIVE_EAGER_OPERATION_JNI_H_
tensorflow/java/src/main/native/eager_session_jni.cc
+63
View File
@@ -0,0 +1,63 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/java/src/main/native/eager_session_jni.h"
#include <cstring>
#include "tensorflow/c/eager/c_api.h"
#include "tensorflow/java/src/main/native/exception_jni.h"
JNIEXPORT jlong JNICALL Java_org_tensorflow_EagerSession_allocate(
JNIEnv* env, jclass clazz, jboolean async, jint dpp, jbyteArray config) {
TFE_ContextOptions* opts = TFE_NewContextOptions();
jbyte* cconfig = nullptr;
TF_Status* status = TF_NewStatus();
if (config != nullptr) {
cconfig = env->GetByteArrayElements(config, nullptr);
TFE_ContextOptionsSetConfig(
opts, cconfig, static_cast<size_t>(env->GetArrayLength(config)),
status);
if (!throwExceptionIfNotOK(env, status)) {
env->ReleaseByteArrayElements(config, cconfig, JNI_ABORT);
TFE_DeleteContextOptions(opts);
TF_DeleteStatus(status);
return 0;
}
}
TFE_ContextOptionsSetAsync(opts, static_cast<unsigned char>(async));
TFE_ContextOptionsSetDevicePlacementPolicy(
opts, static_cast<TFE_ContextDevicePlacementPolicy>(dpp));
TFE_Context* context = TFE_NewContext(opts, status);
TFE_DeleteContextOptions(opts);
if (config != nullptr) {
env->ReleaseByteArrayElements(config, cconfig, JNI_ABORT);
}
if (!throwExceptionIfNotOK(env, status)) {
TF_DeleteStatus(status);
return 0;
}
TF_DeleteStatus(status);
static_assert(sizeof(jlong) >= sizeof(TFE_Context*),
"Cannot represent a C TFE_Op as a Java long");
return reinterpret_cast<jlong>(context);
}
JNIEXPORT void JNICALL Java_org_tensorflow_EagerSession_delete(JNIEnv* env,
jclass clazz,
jlong handle) {
if (handle == 0) return;
TFE_DeleteContext(reinterpret_cast<TFE_Context*>(handle));
}
tensorflow/java/src/main/native/eager_session_jni.h
+44
View File
@@ -0,0 +1,44 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_MAIN_NATIVE_EAGER_SESSION_JNI_H_
#define TENSORFLOW_JAVA_SRC_MAIN_NATIVE_EAGER_SESSION_JNI_H_
#include <jni.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
* Class: org_tensorflow_EagerSession
* Method: allocate
* Signature: (ZI[B)J
*/
JNIEXPORT jlong JNICALL Java_org_tensorflow_EagerSession_allocate(
JNIEnv *env, jclass clazz, jboolean async, jint dpp, jbyteArray config);
/*
* Class: org_tensorflow_EagerSession
* Method: delete
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_EagerSession_delete(JNIEnv *, jclass,
jlong);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif // TENSORFLOW_JAVA_SRC_MAIN_NATIVE_EAGER_SESSION_JNI_H_
tensorflow/java/src/main/native/exception_jni.cc
+75
View File
@@ -0,0 +1,75 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include "tensorflow/c/c_api.h"
#include "tensorflow/java/src/main/native/exception_jni.h"
const char kIllegalArgumentException[] = "java/lang/IllegalArgumentException";
const char kIllegalStateException[] = "java/lang/IllegalStateException";
const char kNullPointerException[] = "java/lang/NullPointerException";
const char kIndexOutOfBoundsException[] = "java/lang/IndexOutOfBoundsException";
const char kUnsupportedOperationException[] =
"java/lang/UnsupportedOperationException";
void throwException(JNIEnv* env, const char* clazz, const char* fmt, ...) {
va_list args;
va_start(args, fmt);
// Using vsnprintf() instead of vasprintf() because the latter doesn't seem to
// be easily available on Windows.
const size_t max_msg_len = 512;
char* message = static_cast<char*>(malloc(max_msg_len));
if (vsnprintf(message, max_msg_len, fmt, args) >= 0) {
env->ThrowNew(env->FindClass(clazz), message);
} else {
env->ThrowNew(env->FindClass(clazz), "");
}
free(message);
va_end(args);
}
namespace {
// Map TF_Codes to unchecked exceptions.
const char* exceptionClassName(TF_Code code) {
switch (code) {
case TF_OK:
return nullptr;
case TF_INVALID_ARGUMENT:
return kIllegalArgumentException;
case TF_UNAUTHENTICATED:
case TF_PERMISSION_DENIED:
return "java/lang/SecurityException";
case TF_RESOURCE_EXHAUSTED:
case TF_FAILED_PRECONDITION:
return kIllegalStateException;
case TF_OUT_OF_RANGE:
return kIndexOutOfBoundsException;
case TF_UNIMPLEMENTED:
return kUnsupportedOperationException;
default:
return "org/tensorflow/TensorFlowException";
}
}
} // namespace
bool throwExceptionIfNotOK(JNIEnv* env, const TF_Status* status) {
const char* clazz = exceptionClassName(TF_GetCode(status));
if (clazz == nullptr) return true;
env->ThrowNew(env->FindClass(clazz), TF_Message(status));
return false;
}
tensorflow/java/src/main/native/exception_jni.h
+42
View File
@@ -0,0 +1,42 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_MAIN_NATIVE_EXCEPTION_JNI_H_
#define TENSORFLOW_JAVA_SRC_MAIN_NATIVE_EXCEPTION_JNI_H_
#include <jni.h>
#ifdef __cplusplus
extern "C" {
#endif
struct TSL_Status;
extern const char kIllegalArgumentException[];
extern const char kIllegalStateException[];
extern const char kNullPointerException[];
extern const char kIndexOutOfBoundsException[];
extern const char kUnsupportedOperationException[];
void throwException(JNIEnv* env, const char* clazz, const char* fmt, ...);
// If status is not TF_OK, then throw an appropriate exception.
// Returns true iff TF_GetCode(status) == TF_OK.
bool throwExceptionIfNotOK(JNIEnv* env, const TSL_Status* status);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif // TENSORFLOW_JAVA_SRC_MAIN_NATIVE_EXCEPTION_JNI_H_
tensorflow/java/src/main/native/graph_jni.cc
+337
View File
@@ -0,0 +1,337 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/java/src/main/native/graph_jni.h"
#include <cstddef>
#include <limits>
#include <memory>
#include "tensorflow/c/c_api.h"
#include "tensorflow/java/src/main/native/exception_jni.h"
#include "tensorflow/java/src/main/native/utils_jni.h"
namespace {
template <class T>
T* requireHandleImpl(JNIEnv* env, jlong handle) {
static_assert(sizeof(jlong) >= sizeof(T*),
"Cannot package C object pointers as a Java long");
if (handle == 0) {
throwException(env, kIllegalStateException,
"close() has been called on the Graph");
return nullptr;
}
return reinterpret_cast<T*>(handle);
}
TF_Graph* requireHandle(JNIEnv* env, jlong handle) {
return requireHandleImpl<TF_Graph>(env, handle);
}
TF_Operation* requireOperationHandle(JNIEnv* env, jlong handle) {
return requireHandleImpl<TF_Operation>(env, handle);
}
} // namespace
JNIEXPORT jlong JNICALL Java_org_tensorflow_Graph_allocate(JNIEnv*, jclass) {
return reinterpret_cast<jlong>(TF_NewGraph());
}
JNIEXPORT void JNICALL Java_org_tensorflow_Graph_delete(JNIEnv*, jclass,
jlong handle) {
if (handle == 0) return;
TF_DeleteGraph(reinterpret_cast<TF_Graph*>(handle));
}
JNIEXPORT jlong JNICALL Java_org_tensorflow_Graph_operation(JNIEnv* env,
jclass clazz,
jlong handle,
jstring name) {
TF_Graph* g = requireHandle(env, handle);
if (g == nullptr) return 0;
const char* cname = env->GetStringUTFChars(name, nullptr);
TF_Operation* op = TF_GraphOperationByName(g, cname);
env->ReleaseStringUTFChars(name, cname);
return reinterpret_cast<jlong>(op);
}
JNIEXPORT jlongArray JNICALL Java_org_tensorflow_Graph_nextOperation(
JNIEnv* env, jclass clazz, jlong handle, jint position) {
TF_Graph* g = requireHandle(env, handle);
if (g == nullptr) return nullptr;
size_t pos = static_cast<size_t>(position);
TF_Operation* operation = TF_GraphNextOperation(g, &pos);
if (operation == nullptr) return nullptr;
jlong handle_and_position[2];
handle_and_position[0] = reinterpret_cast<jlong>(operation);
handle_and_position[1] = static_cast<jlong>(pos);
jlongArray rhett = env->NewLongArray(2);
env->SetLongArrayRegion(rhett, 0, 2, handle_and_position);
return rhett;
}
JNIEXPORT void JNICALL Java_org_tensorflow_Graph_importGraphDef(
JNIEnv* env, jclass clazz, jlong handle, jbyteArray graph_def,
jstring prefix) {
TF_Graph* g = requireHandle(env, handle);
if (g == nullptr) return;
TF_ImportGraphDefOptions* opts = TF_NewImportGraphDefOptions();
jboolean is_copy;
const char* cprefix = env->GetStringUTFChars(prefix, &is_copy);
TF_ImportGraphDefOptionsSetPrefix(opts, cprefix);
env->ReleaseStringUTFChars(prefix, cprefix);
static_assert(sizeof(jbyte) == 1, "unexpected size of the jbyte type");
jbyte* bytes = env->GetByteArrayElements(graph_def, &is_copy);
TF_Buffer* buf =
TF_NewBufferFromString(bytes, env->GetArrayLength(graph_def));
TF_Status* status = TF_NewStatus();
TF_GraphImportGraphDef(g, buf, opts, status);
throwExceptionIfNotOK(env, status);
// Continue cleaning up resources even if an exception was thrown.
TF_DeleteStatus(status);
TF_DeleteBuffer(buf);
env->ReleaseByteArrayElements(graph_def, bytes, JNI_ABORT);
TF_DeleteImportGraphDefOptions(opts);
}
JNIEXPORT jbyteArray JNICALL
Java_org_tensorflow_Graph_toGraphDef(JNIEnv* env, jclass clazz, jlong handle) {
jbyteArray ret = nullptr;
TF_Graph* g = requireHandle(env, handle);
if (g == nullptr) return ret;
TF_Buffer* buf = TF_NewBuffer();
TF_Status* status = TF_NewStatus();
TF_GraphToGraphDef(g, buf, status);
if (throwExceptionIfNotOK(env, status)) {
// sizeof(jsize) is less than sizeof(size_t) on some platforms.
if (buf->length > std::numeric_limits<jint>::max()) {
throwException(env, kIndexOutOfBoundsException,
"GraphDef is too large to serialize into a byte[] array");
} else {
static_assert(sizeof(jbyte) == 1, "unexpected size of the jbyte type");
jint ret_len = static_cast<jint>(buf->length);
ret = env->NewByteArray(ret_len);
env->SetByteArrayRegion(ret, 0, ret_len,
static_cast<const jbyte*>(buf->data));
}
}
TF_DeleteStatus(status);
TF_DeleteBuffer(buf);
return ret;
}
JNIEXPORT jlongArray JNICALL Java_org_tensorflow_Graph_addGradients(
JNIEnv* env, jclass clazz, jlong handle, jstring prefix,
jlongArray y_handles, jintArray y_indices, jlongArray x_handles,
jintArray x_indices, jlongArray dx_handles, jintArray dx_indices) {
TF_Graph* g = requireHandle(env, handle);
if (g == nullptr) return nullptr;
const jint ny = env->GetArrayLength(y_handles);
const jint nx = env->GetArrayLength(x_handles);
std::unique_ptr<TF_Output[]> y(new TF_Output[ny]);
std::unique_ptr<TF_Output[]> x(new TF_Output[nx]);
std::unique_ptr<TF_Output[]> dx(nullptr);
std::unique_ptr<TF_Output[]> dy(new TF_Output[nx]);
resolveOutputs(env, "y", y_handles, y_indices, y.get(), ny);
resolveOutputs(env, "x", x_handles, x_indices, x.get(), nx);
if (dx_handles != nullptr) {
if (env->GetArrayLength(dx_handles) != ny) {
throwException(env, kIllegalArgumentException,
"expected %d, got %d dx handles", ny,
env->GetArrayLength(dx_handles));
}
dx = std::make_unique<TF_Output[]>(ny);
resolveOutputs(env, "dx", dx_handles, dx_indices, dx.get(), ny);
}
if (env->ExceptionCheck()) return nullptr;
const char* cprefix = nullptr;
if (prefix != nullptr) {
cprefix = env->GetStringUTFChars(prefix, nullptr);
}
TF_Status* status = TF_NewStatus();
TF_AddGradientsWithPrefix(g, cprefix, y.get(), ny, x.get(), nx, dx.get(),
status, dy.get());
if (prefix != nullptr) {
env->ReleaseStringUTFChars(prefix, cprefix);
}
if (!throwExceptionIfNotOK(env, status)) {
TF_DeleteStatus(status);
return nullptr;
}
TF_DeleteStatus(status);
// returned array contains both op handles and output indices, in pair
jlongArray dy_handles_and_indices = env->NewLongArray(nx << 1);
jlong* dy_elems = env->GetLongArrayElements(dy_handles_and_indices, nullptr);
for (int i = 0, j = nx; i < nx; ++i, ++j) {
TF_Output dy_output = dy.get()[i];
dy_elems[i] = reinterpret_cast<jlong>(dy_output.oper);
dy_elems[j] = static_cast<jlong>(dy_output.index);
}
env->ReleaseLongArrayElements(dy_handles_and_indices, dy_elems, 0);
return dy_handles_and_indices;
}
// helper function for while loop -- constructs conditional or body subgraph
jlongArray buildSubgraph(JNIEnv* env, jclass clazz, jobject subgraph_builder,
TF_Graph* const subgraph,
const TF_Output* const inputs,
const TF_Output* const outputs, const int ninputs,
const int noutputs) {
jmethodID build_subgraph_method_id = env->GetStaticMethodID(
clazz, "buildSubgraph",
"(Lorg/tensorflow/Graph$WhileSubgraphBuilder;J[J[I[J[I)[J");
if (build_subgraph_method_id == nullptr) return nullptr;
jlong subgraph_handle = reinterpret_cast<jlong>(subgraph);
jlongArray input_handles = env->NewLongArray(ninputs);
jintArray input_indices = env->NewIntArray(ninputs);
jlongArray output_handles = env->NewLongArray(noutputs);
jintArray output_indices = env->NewIntArray(noutputs);
jlong* input_handles_elems =
env->GetLongArrayElements(input_handles, nullptr);
jint* input_indices_elems = env->GetIntArrayElements(input_indices, nullptr);
jlong* output_handles_elems =
env->GetLongArrayElements(output_handles, nullptr);
jint* output_indices_elems =
env->GetIntArrayElements(output_indices, nullptr);
for (int i = 0; i < ninputs; ++i) {
input_handles_elems[i] = reinterpret_cast<jlong>((inputs[i]).oper);
input_indices_elems[i] = static_cast<jint>((inputs[i]).index);
}
for (int i = 0; i < noutputs; ++i) {
output_handles_elems[i] = reinterpret_cast<jlong>((outputs[i]).oper);
output_indices_elems[i] = static_cast<jint>((outputs[i]).index);
}
env->ReleaseLongArrayElements(input_handles, input_handles_elems, 0);
env->ReleaseIntArrayElements(input_indices, input_indices_elems, 0);
env->ReleaseLongArrayElements(output_handles, output_handles_elems, 0);
env->ReleaseIntArrayElements(output_indices, output_indices_elems, 0);
// call Java code to construct the subgraph
jlongArray output_handles_and_indices =
(jlongArray)env->CallStaticObjectMethod(
clazz, build_subgraph_method_id, subgraph_builder, subgraph_handle,
input_handles, input_indices, output_handles, output_indices);
if (env->ExceptionOccurred()) {
env->ExceptionDescribe();
return nullptr;
}
// returned array contains both op handles and output indices, in pair
return output_handles_and_indices;
}
JNIEXPORT jlongArray JNICALL Java_org_tensorflow_Graph_whileLoop(
JNIEnv* env, jclass clazz, jlong handle, jlongArray input_handles,
jintArray input_indices, jstring name, jobject cond_graph_builder,
jobject body_graph_builder) {
TF_Graph* g = requireHandle(env, handle);
TF_Status* status = TF_NewStatus();
if (g == nullptr) return nullptr;
int ninputs = env->GetArrayLength(input_handles);
std::unique_ptr<TF_Output[]> inputs(new TF_Output[ninputs]);
resolveOutputs(env, "inputs", input_handles, input_indices, inputs.get(),
ninputs);
if (env->ExceptionCheck()) return nullptr;
// initialize while params
TF_WhileParams params = TF_NewWhile(g, inputs.get(), ninputs, status);
throwExceptionIfNotOK(env, status);
// build conditional subgraph
jlongArray cond_output_handles_and_indices =
buildSubgraph(env, clazz, cond_graph_builder, params.cond_graph,
params.cond_inputs, &params.cond_output, params.ninputs, 1);
// build body subgraph
jlongArray body_output_handles_and_indices = buildSubgraph(
env, clazz, body_graph_builder, params.body_graph, params.body_inputs,
params.body_outputs, params.ninputs, params.ninputs);
if (cond_output_handles_and_indices == nullptr ||
body_output_handles_and_indices == nullptr)
return nullptr;
// set cond_output param to output of the conditional subgraph
jlong* cond_output_elems =
env->GetLongArrayElements(cond_output_handles_and_indices, nullptr);
TF_Operation* cond_output_op =
requireOperationHandle(env, cond_output_elems[0]);
params.cond_output = {cond_output_op,
static_cast<jint>(cond_output_elems[1])};
env->ReleaseLongArrayElements(cond_output_handles_and_indices,
cond_output_elems, 0);
// set body_outputs param to outputs of the body subgraph
jlong* body_output_elems =
env->GetLongArrayElements(body_output_handles_and_indices, nullptr);
for (int i = 0, j = ninputs; i < ninputs; ++i, ++j) {
TF_Operation* body_output_op =
requireOperationHandle(env, body_output_elems[i]);
params.body_outputs[i] = {body_output_op,
static_cast<jint>(body_output_elems[j])};
}
env->ReleaseLongArrayElements(body_output_handles_and_indices,
body_output_elems, 0);
// set loop name param
params.name = env->GetStringUTFChars(name, nullptr);
// build the while loop, storing loop outputs in `outputs`
std::unique_ptr<TF_Output[]> outputs(new TF_Output[ninputs]);
TF_FinishWhile(&params, status, outputs.get());
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
env->ReleaseStringUTFChars(name, params.name);
// returned array contains both op handles and output indices, in pair
jlongArray output_handles_and_indices = env->NewLongArray(ninputs * 2);
jlong* output_elems =
env->GetLongArrayElements(output_handles_and_indices, nullptr);
for (int i = 0, j = ninputs; i < ninputs; ++i, ++j) {
TF_Output output = outputs.get()[i];
output_elems[i] = reinterpret_cast<jlong>(output.oper);
output_elems[j] = static_cast<jlong>(output.index);
}
env->ReleaseLongArrayElements(output_handles_and_indices, output_elems, 0);
return output_handles_and_indices;
}
tensorflow/java/src/main/native/graph_jni.h
+98
View File
@@ -0,0 +1,98 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_MAIN_NATIVE_GRAPH_JNI_H_
#define TENSORFLOW_JAVA_SRC_MAIN_NATIVE_GRAPH_JNI_H_
#include <jni.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
* Class: org_tensorflow_Graph
* Method: allocate
* Signature: ()J
*/
JNIEXPORT jlong JNICALL Java_org_tensorflow_Graph_allocate(JNIEnv *, jclass);
/*
* Class: org_tensorflow_Graph
* Method: delete
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_Graph_delete(JNIEnv *, jclass,
jlong);
/*
* Class: org_tensorflow_Graph
* Method: operation
* Signature: (JLjava/lang/String;)J
*/
JNIEXPORT jlong JNICALL Java_org_tensorflow_Graph_operation(JNIEnv *, jclass,
jlong, jstring);
/*
* Class: org_tensorflow_Graph
* Method: operations
* Signature: (JI)[J
*/
JNIEXPORT jlongArray JNICALL Java_org_tensorflow_Graph_nextOperation(JNIEnv *,
jclass,
jlong,
jint);
/*
* Class: org_tensorflow_Graph
* Method: importGraphDef
* Signature: (J[BLjava/lang/String;)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_Graph_importGraphDef(JNIEnv *,
jclass, jlong,
jbyteArray,
jstring);
/*
* Class: org_tensorflow_Graph
* Method: toGraphDef
* Signature: (J)[B
*/
JNIEXPORT jbyteArray JNICALL Java_org_tensorflow_Graph_toGraphDef(JNIEnv *,
jclass,
jlong);
/*
* Class: org_tensorflow_Graph
* Method: name
* Signature: (JLjava/lang/String;[J[I[J[I[J[I)[J
*/
JNIEXPORT jlongArray JNICALL Java_org_tensorflow_Graph_addGradients(
JNIEnv *, jclass, jlong, jstring, jlongArray, jintArray, jlongArray,
jintArray, jlongArray, jintArray);
/*
* Class: org_tensorflow_Graph
* Method: whileLoop
* Signature:
* (J[J[IILjava/lang/String;Lorg/tensorflow/Graph/WhileSubgraphBuilder;Lorg/tensorflow/Graph/WhileSubgraphBuilder;)[J
*/
JNIEXPORT jlongArray JNICALL Java_org_tensorflow_Graph_whileLoop(
JNIEnv *, jclass, jlong, jlongArray, jintArray, jstring, jobject, jobject);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif // TENSORFLOW_JAVA_SRC_MAIN_NATIVE_GRAPH_JNI_H_
tensorflow/java/src/main/native/graph_operation_builder_jni.cc
+338
View File
@@ -0,0 +1,338 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/java/src/main/native/graph_operation_builder_jni.h"
#include <cstdint>
#include <cstring>
#include <memory>
#include "tensorflow/c/c_api.h"
#include "tensorflow/java/src/main/native/exception_jni.h"
namespace {
TF_OperationDescription* requireHandle(JNIEnv* env, jlong handle) {
if (handle == 0) {
throwException(env, kIllegalStateException,
"Operation has already been built");
return nullptr;
}
return reinterpret_cast<TF_OperationDescription*>(handle);
}
bool resolveOutput(JNIEnv* env, jlong op_handle, jint index, TF_Output* out) {
if (op_handle == 0) {
throwException(env, kIllegalStateException,
"close() was called on the Graph");
return false;
}
out->oper = reinterpret_cast<TF_Operation*>(op_handle);
out->index = static_cast<int>(index);
return true;
}
TF_Tensor* requireTensor(JNIEnv* env, jlong handle) {
if (handle == 0) {
throwException(env, kIllegalStateException,
"close() has been called on the Tensor");
return nullptr;
}
return reinterpret_cast<TF_Tensor*>(handle);
}
} // namespace
JNIEXPORT jlong JNICALL Java_org_tensorflow_GraphOperationBuilder_allocate(
JNIEnv* env, jclass clazz, jlong graph_handle, jstring type, jstring name) {
if (graph_handle == 0) {
throwException(env, kIllegalStateException,
"close() has been called on the Graph");
return 0;
}
TF_Graph* graph = reinterpret_cast<TF_Graph*>(graph_handle);
const char* op_type = env->GetStringUTFChars(type, nullptr);
const char* op_name = env->GetStringUTFChars(name, nullptr);
TF_OperationDescription* d = TF_NewOperation(graph, op_type, op_name);
env->ReleaseStringUTFChars(name, op_name);
env->ReleaseStringUTFChars(type, op_type);
static_assert(sizeof(jlong) >= sizeof(TF_OperationDescription*),
"Cannot represent a C TF_OperationDescription as a Java long");
return reinterpret_cast<jlong>(d);
}
JNIEXPORT jlong JNICALL Java_org_tensorflow_GraphOperationBuilder_finish(
JNIEnv* env, jclass clazz, jlong handle) {
TF_OperationDescription* d = requireHandle(env, handle);
if (d == nullptr) return 0;
TF_Status* status = TF_NewStatus();
TF_Operation* op = TF_FinishOperation(d, status);
if (throwExceptionIfNotOK(env, status)) {
TF_DeleteStatus(status);
return reinterpret_cast<jlong>(op);
}
TF_DeleteStatus(status);
return 0;
}
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_addInput(
JNIEnv* env, jclass clazz, jlong handle, jlong op_handle, jint index) {
TF_Output out;
if (!resolveOutput(env, op_handle, index, &out)) return;
TF_OperationDescription* d = requireHandle(env, handle);
if (d == nullptr) return;
TF_AddInput(d, out);
}
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_addInputList(
JNIEnv* env, jclass clazz, jlong handle, jlongArray op_handles,
jintArray indices) {
TF_OperationDescription* d = requireHandle(env, handle);
if (d == nullptr) return;
const size_t n = static_cast<size_t>(env->GetArrayLength(op_handles));
if (env->GetArrayLength(indices) != n) {
throwException(env, kIllegalArgumentException,
"mismatch in number of Operations (%d) and output indices "
"(%d) provided",
n, env->GetArrayLength(indices));
return;
}
std::unique_ptr<TF_Output[]> o(new TF_Output[n]);
jlong* oph = env->GetLongArrayElements(op_handles, nullptr);
jint* idx = env->GetIntArrayElements(indices, nullptr);
bool ok = true;
for (int i = 0; i < n && ok; ++i) {
ok = resolveOutput(env, oph[i], idx[i], &o[i]);
}
env->ReleaseIntArrayElements(indices, idx, JNI_ABORT);
env->ReleaseLongArrayElements(op_handles, oph, JNI_ABORT);
if (!ok) return;
TF_AddInputList(d, o.get(), n);
}
JNIEXPORT void JNICALL
Java_org_tensorflow_GraphOperationBuilder_addControlInput(JNIEnv* env,
jclass clazz,
jlong handle,
jlong op_handle) {
if (op_handle == 0) {
throwException(env, kIllegalStateException,
"control input is not valid, "
"perhaps the Graph containing it has been closed()?");
return;
}
TF_Operation* control = reinterpret_cast<TF_Operation*>(op_handle);
TF_OperationDescription* d = requireHandle(env, handle);
if (d == nullptr) return;
TF_AddControlInput(d, control);
}
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_setDevice(
JNIEnv* env, jclass clazz, jlong handle, jstring device) {
TF_OperationDescription* d = requireHandle(env, handle);
if (d == nullptr) return;
const char* cdevice = env->GetStringUTFChars(device, nullptr);
TF_SetDevice(d, cdevice);
env->ReleaseStringUTFChars(device, cdevice);
}
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_setAttrString(
JNIEnv* env, jclass clazz, jlong handle, jstring name, jbyteArray value) {
static_assert(sizeof(jbyte) == 1,
"Require Java byte to be represented as a single byte");
TF_OperationDescription* d = requireHandle(env, handle);
if (d == nullptr) return;
const char* cname = env->GetStringUTFChars(name, nullptr);
jbyte* cvalue = env->GetByteArrayElements(value, nullptr);
TF_SetAttrString(d, cname, cvalue, env->GetArrayLength(value));
env->ReleaseByteArrayElements(value, cvalue, JNI_ABORT);
env->ReleaseStringUTFChars(name, cname);
}
#define DEFINE_SET_ATTR_SCALAR(name, jtype, ctype) \
JNIEXPORT void JNICALL \
Java_org_tensorflow_GraphOperationBuilder_setAttr##name( \
JNIEnv* env, jclass clazz, jlong handle, jstring name, \
jtype value) { \
static_assert( \
sizeof(ctype) >= sizeof(jtype), \
"Information loss when converting between Java and C types"); \
TF_OperationDescription* d = requireHandle(env, handle); \
if (d == nullptr) return; \
const char* cname = env->GetStringUTFChars(name, nullptr); \
TF_SetAttr##name(d, cname, static_cast<ctype>(value)); \
env->ReleaseStringUTFChars(name, cname); \
}
#define DEFINE_SET_ATTR_LIST(name, jname, jtype, ctype) \
JNIEXPORT void JNICALL \
Java_org_tensorflow_GraphOperationBuilder_setAttr##name##List( \
JNIEnv* env, jclass clazz, jlong handle, jstring name, \
jtype##Array value) { \
TF_OperationDescription* d = requireHandle(env, handle); \
if (d == nullptr) return; \
const char* cname = env->GetStringUTFChars(name, nullptr); \
/* Make a copy of the array to paper over any differences */ \
/* in byte representations of the jtype and ctype */ \
/* For example, jint vs TF_DataType. */ \
/* If this copy turns out to be a problem in practice */ \
/* can avoid it for many types. */ \
const int n = env->GetArrayLength(value); \
std::unique_ptr<ctype[]> cvalue(new ctype[n]); \
jtype* elems = env->Get##jname##ArrayElements(value, nullptr); \
for (int i = 0; i < n; ++i) { \
cvalue[i] = static_cast<ctype>(elems[i]); \
} \
TF_SetAttr##name##List(d, cname, cvalue.get(), n); \
env->Release##jname##ArrayElements(value, elems, JNI_ABORT); \
env->ReleaseStringUTFChars(name, cname); \
}
#define DEFINE_SET_ATTR(name, jname, jtype, ctype) \
DEFINE_SET_ATTR_SCALAR(name, jtype, ctype) \
DEFINE_SET_ATTR_LIST(name, jname, jtype, ctype)
DEFINE_SET_ATTR(Int, Long, jlong, int64_t);
DEFINE_SET_ATTR(Float, Float, jfloat, float);
DEFINE_SET_ATTR(Bool, Boolean, jboolean, unsigned char);
DEFINE_SET_ATTR(Type, Int, jint, TF_DataType);
#undef DEFINE_SET_ATTR
#undef DEFINE_SET_ATTR_LIST
#undef DEFINE_SET_ATTR_SCALAR
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_setAttrTensor(
JNIEnv* env, jclass clazz, jlong handle, jstring name,
jlong tensor_handle) {
TF_OperationDescription* d = requireHandle(env, handle);
if (d == nullptr) return;
TF_Tensor* t = requireTensor(env, tensor_handle);
if (t == nullptr) return;
const char* cname = env->GetStringUTFChars(name, nullptr);
TF_Status* status = TF_NewStatus();
TF_SetAttrTensor(d, cname, t, status);
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
env->ReleaseStringUTFChars(name, cname);
}
JNIEXPORT void JNICALL
Java_org_tensorflow_GraphOperationBuilder_setAttrTensorList(
JNIEnv* env, jclass clazz, jlong handle, jstring name,
jlongArray tensor_handles) {
TF_OperationDescription* d = requireHandle(env, handle);
if (d == nullptr) return;
const int n = env->GetArrayLength(tensor_handles);
std::unique_ptr<TF_Tensor*[]> tensors(new TF_Tensor*[n]);
jlong* jhandles = env->GetLongArrayElements(tensor_handles, nullptr);
bool ok = true;
for (int i = 0; i < n && ok; ++i) {
tensors[i] = requireTensor(env, jhandles[i]);
ok = !env->ExceptionCheck();
}
env->ReleaseLongArrayElements(tensor_handles, jhandles, JNI_ABORT);
if (!ok) return;
const char* cname = env->GetStringUTFChars(name, nullptr);
TF_Status* status = TF_NewStatus();
TF_SetAttrTensorList(d, cname, tensors.get(), n, status);
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
env->ReleaseStringUTFChars(name, cname);
}
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_setAttrShape(
JNIEnv* env, jclass clazz, jlong handle, jstring name, jlongArray shape,
jint num_dims) {
TF_OperationDescription* d = requireHandle(env, handle);
if (d == nullptr) return;
std::unique_ptr<int64_t[]> cvalue;
// num_dims and env->GetArrayLength(shape) are assumed to be consistent.
// i.e., either num_dims < 0 or num_dims == env->GetArrayLength(shape).
if (num_dims > 0) {
cvalue.reset(new int64_t[num_dims]);
jlong* elems = env->GetLongArrayElements(shape, nullptr);
for (int i = 0; i < num_dims; ++i) {
cvalue[i] = static_cast<int64_t>(elems[i]);
}
env->ReleaseLongArrayElements(shape, elems, JNI_ABORT);
}
const char* cname = env->GetStringUTFChars(name, nullptr);
TF_SetAttrShape(d, cname, cvalue.get(), static_cast<int>(num_dims));
env->ReleaseStringUTFChars(name, cname);
}
JNIEXPORT void JNICALL
Java_org_tensorflow_GraphOperationBuilder_setAttrShapeList(
JNIEnv* env, jclass clazz, jlong handle, jstring name, jlongArray shapes,
jintArray num_dims) {
TF_OperationDescription* d = requireHandle(env, handle);
if (d == nullptr) return;
std::unique_ptr<int64_t[]> cshapes;
std::unique_ptr<int64_t*[]> cdims;
std::unique_ptr<int[]> cnum_dims;
const int num_dims_length = env->GetArrayLength(num_dims);
if (num_dims_length > 0) {
const int shapes_length = env->GetArrayLength(shapes);
cshapes.reset(new int64_t[shapes_length]);
cdims.reset(new int64_t*[num_dims_length]);
cnum_dims.reset(new int[num_dims_length]);
jlong* shapes_elems =
static_cast<jlong*>(env->GetPrimitiveArrayCritical(shapes, nullptr));
std::memcpy(cshapes.get(), shapes_elems, shapes_length << 3);
env->ReleasePrimitiveArrayCritical(shapes, shapes_elems, JNI_ABORT);
int64_t* cshapes_ptr = cshapes.get();
jint* num_dims_elems =
static_cast<jint*>(env->GetPrimitiveArrayCritical(num_dims, nullptr));
for (int i = 0; i < num_dims_length; ++i) {
cnum_dims[i] = static_cast<int>(num_dims_elems[i]);
cdims[i] = cshapes_ptr;
if (cnum_dims[i] > 0) {
cshapes_ptr += cnum_dims[i];
}
}
env->ReleasePrimitiveArrayCritical(num_dims, num_dims_elems, JNI_ABORT);
}
const char* cname = env->GetStringUTFChars(name, nullptr);
TF_SetAttrShapeList(d, cname, cdims.get(), cnum_dims.get(), num_dims_length);
env->ReleaseStringUTFChars(name, cname);
}
JNIEXPORT void JNICALL
Java_org_tensorflow_GraphOperationBuilder_setAttrStringList(
JNIEnv* env, jclass object, jlong handle, jstring name,
jobjectArray values) {
TF_OperationDescription* d = requireHandle(env, handle);
if (d == nullptr) return;
const char* cname = env->GetStringUTFChars(name, nullptr);
int num_values = env->GetArrayLength(values);
static_assert(sizeof(jbyte) == 1,
"Require Java byte to be represented as a single byte");
std::unique_ptr<jbyteArray[]> jarrays(new jbyteArray[num_values]);
std::unique_ptr<jbyte*[]> jvalues(new jbyte*[num_values]);
std::unique_ptr<void*[]> cvalues(new void*[num_values]);
std::unique_ptr<size_t[]> lengths(new size_t[num_values]);
for (int i = 0; i < num_values; ++i) {
jbyteArray v =
static_cast<jbyteArray>(env->GetObjectArrayElement(values, i));
jarrays[i] = v;
jvalues[i] = env->GetByteArrayElements(v, nullptr);
cvalues[i] = jvalues[i];
lengths[i] = static_cast<size_t>(env->GetArrayLength(v));
}
TF_SetAttrStringList(d, cname, cvalues.get(), lengths.get(), num_values);
for (int i = 0; i < num_values; ++i) {
env->ReleaseByteArrayElements(jarrays[i], jvalues[i], JNI_ABORT);
}
env->ReleaseStringUTFChars(name, cname);
}
tensorflow/java/src/main/native/graph_operation_builder_jni.h
+202
View File
@@ -0,0 +1,202 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_MAIN_NATIVE_GRAPH_OPERATION_BUILDER_JNI_H_
#define TENSORFLOW_JAVA_SRC_MAIN_NATIVE_GRAPH_OPERATION_BUILDER_JNI_H_
#include <jni.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: allocate
* Signature: (JLjava/lang/String;Ljava/lang/String;)J
*/
JNIEXPORT jlong JNICALL Java_org_tensorflow_GraphOperationBuilder_allocate(
JNIEnv *, jclass, jlong, jstring, jstring);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: finish
* Signature: (J)J
*/
JNIEXPORT jlong JNICALL
Java_org_tensorflow_GraphOperationBuilder_finish(JNIEnv *, jclass, jlong);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: addInput
* Signature: (JJI)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_addInput(
JNIEnv *, jclass, jlong, jlong, jint);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: addInputList
* Signature: (J[J[I)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_addInputList(
JNIEnv *, jclass, jlong, jlongArray, jintArray);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: addControlInput
* Signature: (JJ)V
*/
JNIEXPORT void JNICALL
Java_org_tensorflow_GraphOperationBuilder_addControlInput(JNIEnv *, jclass,
jlong, jlong);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: setDevice
* Signature: (JLjava/lang/String;)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_setDevice(
JNIEnv *, jclass, jlong, jstring);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: setAttrString
* Signature: (JLjava/lang/String;[B)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_setAttrString(
JNIEnv *, jclass, jlong, jstring, jbyteArray);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: setAttrInt
* Signature: (JLjava/lang/String;J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_setAttrInt(
JNIEnv *, jclass, jlong, jstring, jlong);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: setAttrIntList
* Signature: (JLjava/lang/String;[J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_setAttrIntList(
JNIEnv *, jclass, jlong, jstring, jlongArray);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: setAttrFloat
* Signature: (JLjava/lang/String;F)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_setAttrFloat(
JNIEnv *, jclass, jlong, jstring, jfloat);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: setAttrFloatList
* Signature: (JLjava/lang/String;[F)V
*/
JNIEXPORT void JNICALL
Java_org_tensorflow_GraphOperationBuilder_setAttrFloatList(JNIEnv *, jclass,
jlong, jstring,
jfloatArray);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: setAttrBool
* Signature: (JLjava/lang/String;Z)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_setAttrBool(
JNIEnv *, jclass, jlong, jstring, jboolean);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: setAttrBoolList
* Signature: (JLjava/lang/String;[Z)V
*/
JNIEXPORT void JNICALL
Java_org_tensorflow_GraphOperationBuilder_setAttrBoolList(JNIEnv *, jclass,
jlong, jstring,
jbooleanArray);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: setAttrType
* Signature: (JLjava/lang/String;I)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_setAttrType(
JNIEnv *, jclass, jlong, jstring, jint);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: setAttrTypeList
* Signature: (JLjava/lang/String;[I)V
*/
JNIEXPORT void JNICALL
Java_org_tensorflow_GraphOperationBuilder_setAttrTypeList(JNIEnv *, jclass,
jlong, jstring,
jintArray);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: setAttrTensor
* Signature: (JLjava/lang/String;J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_setAttrTensor(
JNIEnv *, jclass, jlong, jstring, jlong);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: setAttrTensorList
* Signature: (JLjava/lang/String;[J)V
*/
JNIEXPORT void JNICALL
Java_org_tensorflow_GraphOperationBuilder_setAttrTensorList(JNIEnv *, jclass,
jlong, jstring,
jlongArray);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: setAttrShape
* Signature: (JLjava/lang/String;[JI)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_GraphOperationBuilder_setAttrShape(
JNIEnv *, jclass, jlong, jstring, jlongArray, jint);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: setAttrShapeList
* Signature: (JLjava/lang/String;[J[I)V
*/
JNIEXPORT void JNICALL
Java_org_tensorflow_GraphOperationBuilder_setAttrShapeList(JNIEnv *, jclass,
jlong, jstring,
jlongArray,
jintArray);
/*
* Class: org_tensorflow_GraphOperationBuilder
* Method: setAttrStringList
* Signature: (JLjava/lang/String;[L)V
*/
JNIEXPORT void JNICALL
Java_org_tensorflow_GraphOperationBuilder_setAttrStringList(JNIEnv *, jclass,
jlong, jstring,
jobjectArray);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif // TENSORFLOW_JAVA_SRC_MAIN_NATIVE_GRAPH_OPERATION_BUILDER_JNI_H_
tensorflow/java/src/main/native/graph_operation_jni.cc
+168
View File
@@ -0,0 +1,168 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/java/src/main/native/graph_operation_jni.h"
#include <cstdint>
#include <memory>
#include "tensorflow/c/c_api.h"
#include "tensorflow/java/src/main/native/exception_jni.h"
namespace {
template <class T>
T* requireHandleImpl(JNIEnv* env, jlong handle) {
static_assert(sizeof(jlong) >= sizeof(T*),
"Cannot package C object pointers as a Java long");
if (handle == 0) {
throwException(
env, kNullPointerException,
"close() has been called on the Graph this Operation was a part of");
return nullptr;
}
return reinterpret_cast<T*>(handle);
}
TF_Operation* requireHandle(JNIEnv* env, jlong handle) {
return requireHandleImpl<TF_Operation>(env, handle);
}
TF_Graph* requireGraphHandle(JNIEnv* env, jlong handle) {
return requireHandleImpl<TF_Graph>(env, handle);
}
} // namespace
JNIEXPORT jstring JNICALL Java_org_tensorflow_GraphOperation_name(
JNIEnv* env, jclass clazz, jlong handle) {
TF_Operation* op = requireHandle(env, handle);
if (op == nullptr) return nullptr;
return env->NewStringUTF(TF_OperationName(op));
}
JNIEXPORT jstring JNICALL Java_org_tensorflow_GraphOperation_type(
JNIEnv* env, jclass clazz, jlong handle) {
TF_Operation* op = requireHandle(env, handle);
if (op == nullptr) return nullptr;
return env->NewStringUTF(TF_OperationOpType(op));
}
JNIEXPORT jint JNICALL Java_org_tensorflow_GraphOperation_numOutputs(
JNIEnv* env, jclass clazz, jlong handle) {
TF_Operation* op = requireHandle(env, handle);
if (op == nullptr) return 0;
return TF_OperationNumOutputs(op);
}
JNIEXPORT jint JNICALL Java_org_tensorflow_GraphOperation_outputListLength(
JNIEnv* env, jclass clazz, jlong handle, jstring name) {
TF_Operation* op = requireHandle(env, handle);
if (op == nullptr) return 0;
TF_Status* status = TF_NewStatus();
const char* cname = env->GetStringUTFChars(name, nullptr);
int result = TF_OperationOutputListLength(op, cname, status);
env->ReleaseStringUTFChars(name, cname);
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
return result;
}
JNIEXPORT jlongArray JNICALL Java_org_tensorflow_GraphOperation_shape(
JNIEnv* env, jclass clazz, jlong graph_handle, jlong op_handle,
jint output_index) {
TF_Graph* graph = requireGraphHandle(env, graph_handle);
if (graph == nullptr) return nullptr;
TF_Operation* op = requireHandle(env, op_handle);
if (op == nullptr) return nullptr;
int num_outputs = TF_OperationNumOutputs(op);
if (output_index < 0 || output_index >= num_outputs) {
throwException(
env, kIndexOutOfBoundsException,
"invalid output index (%d) for an operation that has %d outputs",
output_index, num_outputs);
return nullptr;
}
TF_Output output{op, output_index};
TF_Status* status = TF_NewStatus();
jsize num_dims = TF_GraphGetTensorNumDims(graph, output, status);
if (!throwExceptionIfNotOK(env, status) || num_dims < 0) {
TF_DeleteStatus(status);
return nullptr;
}
static_assert(sizeof(jlong) == sizeof(int64_t),
"Java long is not compatible with the TensorFlow C API");
// One might have trivially wanted to do:
// TF_GraphGetTensorShape(graph, output, static_cast<int64_t*>(dims), ...)
// but on some platforms this fails with:
// static_cast from 'jlong *' (aka 'long *') to 'int64_t *' (aka 'long long
// *') is not allowed
// For now, do the expensive but safe thing of copying.
std::unique_ptr<int64_t[]> cdims(new int64_t[num_dims]);
TF_GraphGetTensorShape(graph, output, cdims.get(), static_cast<int>(num_dims),
status);
if (!throwExceptionIfNotOK(env, status)) {
TF_DeleteStatus(status);
return nullptr;
}
TF_DeleteStatus(status);
jlongArray ret = env->NewLongArray(num_dims);
jlong* dims = env->GetLongArrayElements(ret, nullptr);
for (int i = 0; i < num_dims; ++i) {
dims[i] = static_cast<jlong>(cdims[i]);
}
env->ReleaseLongArrayElements(ret, dims, 0);
return ret;
}
JNIEXPORT jint JNICALL Java_org_tensorflow_GraphOperation_dtype(
JNIEnv* env, jclass clazz, jlong graph_handle, jlong op_handle,
jint output_index) {
TF_Graph* graph = requireGraphHandle(env, graph_handle);
if (graph == nullptr) return 0;
TF_Operation* op = requireHandle(env, op_handle);
if (op == nullptr) return 0;
int num_outputs = TF_OperationNumOutputs(op);
if (output_index < 0 || output_index >= num_outputs) {
throwException(
env, kIndexOutOfBoundsException,
"invalid output index (%d) for an operation that has %d outputs",
output_index, num_outputs);
return 0;
}
return static_cast<jint>(TF_OperationOutputType(TF_Output{op, output_index}));
}
JNIEXPORT jint JNICALL Java_org_tensorflow_GraphOperation_inputListLength(
JNIEnv* env, jclass clazz, jlong handle, jstring name) {
TF_Operation* op = requireHandle(env, handle);
if (op == nullptr) return 0;
TF_Status* status = TF_NewStatus();
const char* cname = env->GetStringUTFChars(name, nullptr);
int result = TF_OperationInputListLength(op, cname, status);
env->ReleaseStringUTFChars(name, cname);
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
return result;
}
tensorflow/java/src/main/native/graph_operation_jni.h
+88
View File
@@ -0,0 +1,88 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_MAIN_NATIVE_GRAPH_OPERATION_JNI_H_
#define TENSORFLOW_JAVA_SRC_MAIN_NATIVE_GRAPH_OPERATION_JNI_H_
#include <jni.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
* Class: org_tensorflow_GraphOperation
* Method: name
* Signature: (J)Ljava/lang/String;
*/
JNIEXPORT jstring JNICALL Java_org_tensorflow_GraphOperation_name(JNIEnv *,
jclass,
jlong);
/*
* Class: org_tensorflow_GraphOperation
* Method: type
* Signature: (J)Ljava/lang/String;
*/
JNIEXPORT jstring JNICALL Java_org_tensorflow_GraphOperation_type(JNIEnv *,
jclass,
jlong);
/*
* Class: org_tensorflow_GraphOperation
* Method: numOutputs
* Signature: (J)I
*/
JNIEXPORT jint JNICALL Java_org_tensorflow_GraphOperation_numOutputs(JNIEnv *,
jclass,
jlong);
/*
* Class: org_tensorflow_GraphOperation
* Method: outputListLength
* Signature: (JLjava/lang/String;)I
*/
JNIEXPORT jint JNICALL Java_org_tensorflow_GraphOperation_outputListLength(
JNIEnv *, jclass, jlong, jstring);
/*
* Class: org_tensorflow_GraphOperation
* Method: shape
* Signature: (JJI)[J
*/
JNIEXPORT jlongArray JNICALL
Java_org_tensorflow_GraphOperation_shape(JNIEnv *, jclass, jlong, jlong, jint);
/*
* Class: org_tensorflow_GraphOperation
* Method: dtype
* Signature: (JJI)I
*/
JNIEXPORT jint JNICALL Java_org_tensorflow_GraphOperation_dtype(JNIEnv *,
jclass, jlong,
jlong, jint);
/*
* Class: org_tensorflow_GraphOperation
* Method: inputListLength
* Signature: (JLjava/lang/String;)I
*/
JNIEXPORT jint JNICALL Java_org_tensorflow_GraphOperation_inputListLength(
JNIEnv *, jclass, jlong, jstring);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif // TENSORFLOW_JAVA_SRC_MAIN_NATIVE_GRAPH_OPERATION_JNI_H_
tensorflow/java/src/main/native/saved_model_bundle_jni.cc
+122
View File
@@ -0,0 +1,122 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/java/src/main/native/saved_model_bundle_jni.h"
#include <cstddef>
#include <limits>
#include <memory>
#include "tensorflow/c/c_api.h"
#include "tensorflow/java/src/main/native/exception_jni.h"
JNIEXPORT jobject JNICALL Java_org_tensorflow_SavedModelBundle_load(
JNIEnv* env, jclass clazz, jstring export_dir, jobjectArray tags,
jbyteArray config, jbyteArray run_options) {
TF_Status* status = TF_NewStatus();
jobject bundle = nullptr;
// allocate parameters for TF_LoadSessionFromSavedModel
TF_SessionOptions* opts = TF_NewSessionOptions();
if (config != nullptr) {
size_t sz = env->GetArrayLength(config);
if (sz > 0) {
jbyte* config_data = env->GetByteArrayElements(config, nullptr);
TF_SetConfig(opts, static_cast<void*>(config_data), sz, status);
env->ReleaseByteArrayElements(config, config_data, JNI_ABORT);
if (!throwExceptionIfNotOK(env, status)) {
TF_DeleteSessionOptions(opts);
TF_DeleteStatus(status);
return nullptr;
}
}
}
TF_Buffer* crun_options = nullptr;
if (run_options != nullptr) {
size_t sz = env->GetArrayLength(run_options);
if (sz > 0) {
jbyte* run_options_data = env->GetByteArrayElements(run_options, nullptr);
crun_options =
TF_NewBufferFromString(static_cast<void*>(run_options_data), sz);
env->ReleaseByteArrayElements(run_options, run_options_data, JNI_ABORT);
}
}
const char* cexport_dir = env->GetStringUTFChars(export_dir, nullptr);
std::unique_ptr<const char* []> tags_ptrs;
size_t tags_len = env->GetArrayLength(tags);
tags_ptrs.reset(new const char*[tags_len]);
for (size_t i = 0; i < tags_len; ++i) {
jstring tag = static_cast<jstring>(env->GetObjectArrayElement(tags, i));
tags_ptrs[i] = env->GetStringUTFChars(tag, nullptr);
env->DeleteLocalRef(tag);
}
// load the session
TF_Graph* graph = TF_NewGraph();
TF_Buffer* metagraph_def = TF_NewBuffer();
TF_Session* session = TF_LoadSessionFromSavedModel(
opts, crun_options, cexport_dir, tags_ptrs.get(), tags_len, graph,
metagraph_def, status);
// release the parameters
TF_DeleteSessionOptions(opts);
if (crun_options != nullptr) {
TF_DeleteBuffer(crun_options);
}
env->ReleaseStringUTFChars(export_dir, cexport_dir);
for (size_t i = 0; i < tags_len; ++i) {
jstring tag = static_cast<jstring>(env->GetObjectArrayElement(tags, i));
env->ReleaseStringUTFChars(tag, tags_ptrs[i]);
env->DeleteLocalRef(tag);
}
// handle the result
if (throwExceptionIfNotOK(env, status)) {
// sizeof(jsize) is less than sizeof(size_t) on some platforms.
if (metagraph_def->length > std::numeric_limits<jint>::max()) {
throwException(
env, kIndexOutOfBoundsException,
"MetaGraphDef is too large to serialize into a byte[] array");
} else {
static_assert(sizeof(jbyte) == 1, "unexpected size of the jbyte type");
jint jmetagraph_len = static_cast<jint>(metagraph_def->length);
jbyteArray jmetagraph_def = env->NewByteArray(jmetagraph_len);
env->SetByteArrayRegion(jmetagraph_def, 0, jmetagraph_len,
static_cast<const jbyte*>(metagraph_def->data));
jmethodID method = env->GetStaticMethodID(
clazz, "fromHandle", "(JJ[B)Lorg/tensorflow/SavedModelBundle;");
bundle = env->CallStaticObjectMethod(
clazz, method, reinterpret_cast<jlong>(graph),
reinterpret_cast<jlong>(session), jmetagraph_def);
graph = nullptr;
session = nullptr;
env->DeleteLocalRef(jmetagraph_def);
}
}
if (session != nullptr) {
TF_CloseSession(session, status);
// Result of close is ignored, delete anyway.
TF_DeleteSession(session, status);
}
if (graph != nullptr) {
TF_DeleteGraph(graph);
}
TF_DeleteBuffer(metagraph_def);
TF_DeleteStatus(status);
return bundle;
}
tensorflow/java/src/main/native/saved_model_bundle_jni.h
+37
View File
@@ -0,0 +1,37 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_MAIN_NATIVE_SAVED_MODEL_BUNDLE_JNI_H_
#define TENSORFLOW_JAVA_SRC_MAIN_NATIVE_SAVED_MODEL_BUNDLE_JNI_H_
#include <jni.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
* Class: org_tensorflow_SavedModelBundle
* Method: load
* Signature:
* (Ljava/lang/String;[Ljava/lang/String;[B;[B)Lorg/tensorflow/SavedModelBundle;
*/
JNIEXPORT jobject JNICALL Java_org_tensorflow_SavedModelBundle_load(
JNIEnv *, jclass, jstring, jobjectArray, jbyteArray, jbyteArray);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif // TENSORFLOW_JAVA_SRC_MAIN_NATIVE_SAVED_MODEL_BUNDLE_JNI_H_
tensorflow/java/src/main/native/server_jni.cc
+107
View File
@@ -0,0 +1,107 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/java/src/main/native/server_jni.h"
#include <cstddef>
#include "tensorflow/c/c_api.h"
#include "tensorflow/java/src/main/native/exception_jni.h"
#include "tensorflow/java/src/main/native/utils_jni.h"
namespace {
TF_Server* requireHandle(JNIEnv* env, jlong handle) {
static_assert(sizeof(jlong) >= sizeof(TF_Server*),
"Cannot package C object pointers as a Java long");
if (handle == 0) {
throwException(env, kIllegalStateException,
"close() has been called on the Server");
return nullptr;
}
return reinterpret_cast<TF_Server*>(handle);
}
} // namespace
JNIEXPORT jlong JNICALL Java_org_tensorflow_Server_allocate(
JNIEnv* env, jclass clazz, jbyteArray server_def) {
TF_Status* status = TF_NewStatus();
jbyte* server_def_ptr = env->GetByteArrayElements(server_def, nullptr);
TF_Server* server = TF_NewServer(
server_def_ptr, static_cast<size_t>(env->GetArrayLength(server_def)),
status);
env->ReleaseByteArrayElements(server_def, server_def_ptr, JNI_ABORT);
bool ok = throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
return ok ? reinterpret_cast<jlong>(server) : 0;
}
JNIEXPORT void JNICALL Java_org_tensorflow_Server_start(JNIEnv* env,
jclass clazz,
jlong handle) {
TF_Server* server = requireHandle(env, handle);
if (server == nullptr) return;
TF_Status* status = TF_NewStatus();
TF_ServerStart(server, status);
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
}
JNIEXPORT void JNICALL Java_org_tensorflow_Server_stop(JNIEnv* env,
jclass clazz,
jlong handle) {
TF_Server* server = requireHandle(env, handle);
if (server == nullptr) return;
TF_Status* status = TF_NewStatus();
TF_ServerStop(server, status);
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
}
JNIEXPORT void JNICALL Java_org_tensorflow_Server_join(JNIEnv* env,
jclass clazz,
jlong handle) {
TF_Server* server = requireHandle(env, handle);
if (server == nullptr) return;
TF_Status* status = TF_NewStatus();
TF_ServerJoin(server, status);
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
}
JNIEXPORT void JNICALL Java_org_tensorflow_Server_delete(JNIEnv* env,
jclass clazz,
jlong handle) {
TF_Server* server = requireHandle(env, handle);
if (server == nullptr) return;
TF_DeleteServer(server);
}
tensorflow/java/src/main/native/server_jni.h
+66
View File
@@ -0,0 +1,66 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_MAIN_NATIVE_SERVER_JNI_H_
#define TENSORFLOW_JAVA_SRC_MAIN_NATIVE_SERVER_JNI_H_
#include <jni.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
* Class: org_tensorflow_Server
* Method: allocate
* Signature: ([B)J
*/
JNIEXPORT jlong JNICALL
Java_org_tensorflow_Server_allocate(JNIEnv *, jclass, jbyteArray server_def);
/*
* Class: org_tensorflow_Server
* Method: start
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_Server_start(JNIEnv *, jclass,
jlong);
/*
* Class: org_tensorflow_Server
* Method: stop
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_Server_stop(JNIEnv *, jclass, jlong);
/*
* Class: org_tensorflow_Session
* Method: join
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_Server_join(JNIEnv *, jclass, jlong);
/*
* Class: org_tensorflow_Session
* Method: delete
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_Server_delete(JNIEnv *, jclass,
jlong);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif // TENSORFLOW_JAVA_SRC_MAIN_NATIVE_SERVER_JNI_H_
tensorflow/java/src/main/native/session_jni.cc
+203
View File
@@ -0,0 +1,203 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <string.h>
#include <memory>
#include "tensorflow/c/c_api.h"
#include "tensorflow/java/src/main/native/utils_jni.h"
#include "tensorflow/java/src/main/native/exception_jni.h"
#include "tensorflow/java/src/main/native/session_jni.h"
namespace {
TF_Session* requireHandle(JNIEnv* env, jlong handle) {
static_assert(sizeof(jlong) >= sizeof(TF_Session*),
"Cannot package C object pointers as a Java long");
if (handle == 0) {
throwException(env, kNullPointerException,
"close() has been called on the Session");
return nullptr;
}
return reinterpret_cast<TF_Session*>(handle);
}
template <class T>
void resolveHandles(JNIEnv* env, const char* type, jlongArray src_array,
T** dst, jint n) {
if (env->ExceptionCheck()) return;
jint len = env->GetArrayLength(src_array);
if (len != n) {
throwException(env, kIllegalArgumentException, "expected %d, got %d %s", n,
len, type);
return;
}
jlong* src_start = env->GetLongArrayElements(src_array, nullptr);
jlong* src = src_start;
for (int i = 0; i < n; ++i, ++src, ++dst) {
if (*src == 0) {
throwException(env, kNullPointerException, "invalid %s (#%d of %d)", type,
i, n);
break;
}
*dst = reinterpret_cast<T*>(*src);
}
env->ReleaseLongArrayElements(src_array, src_start, JNI_ABORT);
}
void TF_MaybeDeleteBuffer(TF_Buffer* buf) {
if (buf == nullptr) return;
TF_DeleteBuffer(buf);
}
typedef std::unique_ptr<TF_Buffer, decltype(&TF_MaybeDeleteBuffer)>
unique_tf_buffer;
unique_tf_buffer MakeUniqueBuffer(TF_Buffer* buf) {
return unique_tf_buffer(buf, TF_MaybeDeleteBuffer);
}
} // namespace
JNIEXPORT jlong JNICALL Java_org_tensorflow_Session_allocate(
JNIEnv* env, jclass clazz, jlong graph_handle) {
return Java_org_tensorflow_Session_allocate2(env, clazz, graph_handle,
nullptr, nullptr);
}
JNIEXPORT jlong JNICALL Java_org_tensorflow_Session_allocate2(
JNIEnv* env, jclass clazz, jlong graph_handle, jstring target,
jbyteArray config) {
if (graph_handle == 0) {
throwException(env, kNullPointerException, "Graph has been close()d");
return 0;
}
TF_Graph* graph = reinterpret_cast<TF_Graph*>(graph_handle);
TF_Status* status = TF_NewStatus();
TF_SessionOptions* opts = TF_NewSessionOptions();
jbyte* cconfig = nullptr;
if (config != nullptr) {
cconfig = env->GetByteArrayElements(config, nullptr);
TF_SetConfig(opts, cconfig,
static_cast<size_t>(env->GetArrayLength(config)), status);
if (!throwExceptionIfNotOK(env, status)) {
env->ReleaseByteArrayElements(config, cconfig, JNI_ABORT);
TF_DeleteSessionOptions(opts);
TF_DeleteStatus(status);
return 0;
}
}
const char* ctarget = nullptr;
if (target != nullptr) {
ctarget = env->GetStringUTFChars(target, nullptr);
}
TF_Session* session = TF_NewSession(graph, opts, status);
if (config != nullptr) {
env->ReleaseByteArrayElements(config, cconfig, JNI_ABORT);
}
if (target != nullptr) {
env->ReleaseStringUTFChars(target, ctarget);
}
TF_DeleteSessionOptions(opts);
bool ok = throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
return ok ? reinterpret_cast<jlong>(session) : 0;
}
JNIEXPORT void JNICALL Java_org_tensorflow_Session_delete(JNIEnv* env,
jclass clazz,
jlong handle) {
TF_Session* session = requireHandle(env, handle);
if (session == nullptr) return;
TF_Status* status = TF_NewStatus();
TF_CloseSession(session, status);
// Result of close is ignored, delete anyway.
TF_DeleteSession(session, status);
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
}
JNIEXPORT jbyteArray JNICALL Java_org_tensorflow_Session_run(
JNIEnv* env, jclass clazz, jlong handle, jbyteArray jrun_options,
jlongArray input_tensor_handles, jlongArray input_op_handles,
jintArray input_op_indices, jlongArray output_op_handles,
jintArray output_op_indices, jlongArray target_op_handles,
jboolean want_run_metadata, jlongArray output_tensor_handles) {
TF_Session* session = requireHandle(env, handle);
if (session == nullptr) return nullptr;
const jint ninputs = env->GetArrayLength(input_tensor_handles);
const jint noutputs = env->GetArrayLength(output_tensor_handles);
const jint ntargets = env->GetArrayLength(target_op_handles);
std::unique_ptr<TF_Output[]> inputs(new TF_Output[ninputs]);
std::unique_ptr<TF_Tensor* []> input_values(new TF_Tensor*[ninputs]);
std::unique_ptr<TF_Output[]> outputs(new TF_Output[noutputs]);
std::unique_ptr<TF_Tensor* []> output_values(new TF_Tensor*[noutputs]);
std::unique_ptr<TF_Operation* []> targets(new TF_Operation*[ntargets]);
unique_tf_buffer run_metadata(
MakeUniqueBuffer(want_run_metadata ? TF_NewBuffer() : nullptr));
resolveHandles(env, "input Tensors", input_tensor_handles, input_values.get(),
ninputs);
resolveOutputs(env, "input", input_op_handles, input_op_indices, inputs.get(),
ninputs);
resolveOutputs(env, "output", output_op_handles, output_op_indices,
outputs.get(), noutputs);
resolveHandles(env, "target Operations", target_op_handles, targets.get(),
ntargets);
if (env->ExceptionCheck()) return nullptr;
TF_Status* status = TF_NewStatus();
unique_tf_buffer run_options(MakeUniqueBuffer(nullptr));
jbyte* jrun_options_data = nullptr;
if (jrun_options != nullptr) {
size_t sz = env->GetArrayLength(jrun_options);
if (sz > 0) {
jrun_options_data = env->GetByteArrayElements(jrun_options, nullptr);
run_options.reset(
TF_NewBufferFromString(static_cast<void*>(jrun_options_data), sz));
}
}
TF_SessionRun(session, run_options.get(), inputs.get(), input_values.get(),
static_cast<int>(ninputs), outputs.get(), output_values.get(),
static_cast<int>(noutputs), targets.get(),
static_cast<int>(ntargets), run_metadata.get(), status);
if (jrun_options_data != nullptr) {
env->ReleaseByteArrayElements(jrun_options, jrun_options_data, JNI_ABORT);
}
if (!throwExceptionIfNotOK(env, status)) {
TF_DeleteStatus(status);
return nullptr;
}
jlong* t = env->GetLongArrayElements(output_tensor_handles, nullptr);
for (int i = 0; i < noutputs; ++i) {
t[i] = reinterpret_cast<jlong>(output_values[i]);
}
env->ReleaseLongArrayElements(output_tensor_handles, t, 0);
jbyteArray ret = nullptr;
if (run_metadata != nullptr) {
ret = env->NewByteArray(run_metadata->length);
env->SetByteArrayRegion(ret, 0, run_metadata->length,
reinterpret_cast<const jbyte*>(run_metadata->data));
}
TF_DeleteStatus(status);
return ret;
}
tensorflow/java/src/main/native/session_jni.h
+62
View File
@@ -0,0 +1,62 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_MAIN_NATIVE_SESSION_JNI_H_
#define TENSORFLOW_JAVA_SRC_MAIN_NATIVE_SESSION_JNI_H_
#include <jni.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
* Class: org_tensorflow_Session
* Method: allocate
* Signature: (J)J
*/
JNIEXPORT jlong JNICALL Java_org_tensorflow_Session_allocate(JNIEnv *, jclass,
jlong);
/*
* Class: org_tensorflow_Session
* Method: allocate2
* Signature: (JLjava/lang/String;[B)J
*/
JNIEXPORT jlong JNICALL Java_org_tensorflow_Session_allocate2(JNIEnv *, jclass,
jlong, jstring,
jbyteArray);
/*
* Class: org_tensorflow_Session
* Method: delete
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_Session_delete(JNIEnv *, jclass,
jlong);
/*
* Class: org_tensorflow_Session
* Method: run
* Signature: (J[B[J[J[I[J[I[JZ[J)[B
*/
JNIEXPORT jbyteArray JNICALL Java_org_tensorflow_Session_run(
JNIEnv *, jclass, jlong, jbyteArray, jlongArray, jlongArray, jintArray,
jlongArray, jintArray, jlongArray, jboolean, jlongArray);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif // TENSORFLOW_JAVA_SRC_MAIN_NATIVE_SESSION_JNI_H_
tensorflow/java/src/main/native/tensor_jni.cc
+564
View File
@@ -0,0 +1,564 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/java/src/main/native/tensor_jni.h"
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <cstdint>
#include "tensorflow/c/c_api.h"
#include "tensorflow/java/src/main/native/exception_jni.h"
namespace {
TF_Tensor* requireHandle(JNIEnv* env, jlong handle) {
if (handle == 0) {
throwException(env, kNullPointerException,
"close() was called on the Tensor");
return nullptr;
}
return reinterpret_cast<TF_Tensor*>(handle);
}
size_t elemByteSize(TF_DataType dtype) {
// The code in this file makes the assumption that the
// TensorFlow TF_DataTypes and the Java primitive types
// have the same byte sizes. Validate that:
switch (dtype) {
case TF_BOOL:
case TF_UINT8:
static_assert(sizeof(jboolean) == 1,
"Java boolean not compatible with TF_BOOL");
static_assert(sizeof(jbyte) == 1,
"Java byte not compatible with TF_UINT8");
return 1;
case TF_FLOAT:
case TF_INT32:
static_assert(sizeof(jfloat) == 4,
"Java float not compatible with TF_FLOAT");
static_assert(sizeof(jint) == 4, "Java int not compatible with TF_INT32");
return 4;
case TF_DOUBLE:
case TF_INT64:
static_assert(sizeof(jdouble) == 8,
"Java double not compatible with TF_DOUBLE");
static_assert(sizeof(jlong) == 8,
"Java long not compatible with TF_INT64");
return 8;
default:
return 0;
}
}
// Write a Java scalar object (java.lang.Integer etc.) to a TF_Tensor.
void writeScalar(JNIEnv* env, jobject src, TF_DataType dtype, void* dst,
size_t dst_size) {
size_t sz = elemByteSize(dtype);
if (sz != dst_size) {
throwException(
env, kIllegalStateException,
"scalar (%d bytes) not compatible with allocated tensor (%d bytes)", sz,
dst_size);
return;
}
switch (dtype) {
// env->FindClass and env->GetMethodID are expensive and JNI best practices
// suggest that they should be cached. However, until the creation of scalar
// valued tensors seems to become a noticeable fraction of program execution,
// ignore that cost.
#define CASE(dtype, jtype, method_name, method_signature, call_type) \
case dtype: { \
jclass clazz = env->FindClass("java/lang/Number"); \
jmethodID method = env->GetMethodID(clazz, method_name, method_signature); \
jtype v = env->Call##call_type##Method(src, method); \
memcpy(dst, &v, sz); \
return; \
}
CASE(TF_FLOAT, jfloat, "floatValue", "()F", Float);
CASE(TF_DOUBLE, jdouble, "doubleValue", "()D", Double);
CASE(TF_INT32, jint, "intValue", "()I", Int);
CASE(TF_INT64, jlong, "longValue", "()J", Long);
CASE(TF_UINT8, jbyte, "byteValue", "()B", Byte);
#undef CASE
case TF_BOOL: {
jclass clazz = env->FindClass("java/lang/Boolean");
jmethodID method = env->GetMethodID(clazz, "booleanValue", "()Z");
jboolean v = env->CallBooleanMethod(src, method);
*(static_cast<unsigned char*>(dst)) = v ? 1 : 0;
return;
}
default:
throwException(env, kIllegalStateException, "invalid DataType(%d)",
dtype);
return;
}
}
// Copy a 1-D array of Java primitive types to the tensor buffer dst.
// Returns the number of bytes written to dst.
size_t write1DArray(JNIEnv* env, jarray array, TF_DataType dtype, void* dst,
size_t dst_size) {
const int nelems = env->GetArrayLength(array);
jboolean is_copy;
switch (dtype) {
#define CASE(dtype, jtype, get_type) \
case dtype: { \
jtype##Array a = static_cast<jtype##Array>(array); \
jtype* values = env->Get##get_type##ArrayElements(a, &is_copy); \
size_t to_copy = nelems * elemByteSize(dtype); \
if (to_copy > dst_size) { \
throwException( \
env, kIllegalStateException, \
"cannot write Java array of %d bytes to Tensor of %d bytes", \
to_copy, dst_size); \
to_copy = 0; \
} else { \
memcpy(dst, values, to_copy); \
} \
env->Release##get_type##ArrayElements(a, values, JNI_ABORT); \
return to_copy; \
}
CASE(TF_FLOAT, jfloat, Float);
CASE(TF_DOUBLE, jdouble, Double);
CASE(TF_INT32, jint, Int);
CASE(TF_INT64, jlong, Long);
CASE(TF_BOOL, jboolean, Boolean);
CASE(TF_UINT8, jbyte, Byte);
#undef CASE
default:
throwException(env, kIllegalStateException, "invalid DataType(%d)",
dtype);
return 0;
}
}
// Copy the elements of a 1-D array from the tensor buffer src to a 1-D array of
// Java primitive types. Returns the number of bytes read from src.
size_t read1DArray(JNIEnv* env, TF_DataType dtype, const void* src,
size_t src_size, jarray dst) {
const int len = env->GetArrayLength(dst);
const size_t sz = len * elemByteSize(dtype);
if (sz > src_size) {
throwException(
env, kIllegalStateException,
"cannot fill a Java array of %d bytes with a Tensor of %d bytes", sz,
src_size);
return 0;
}
switch (dtype) {
#define CASE(dtype, jtype, primitive_type) \
case dtype: { \
jtype##Array arr = static_cast<jtype##Array>(dst); \
env->Set##primitive_type##ArrayRegion(arr, 0, len, \
static_cast<const jtype*>(src)); \
return sz; \
}
CASE(TF_FLOAT, jfloat, Float);
CASE(TF_DOUBLE, jdouble, Double);
CASE(TF_INT32, jint, Int);
CASE(TF_INT64, jlong, Long);
CASE(TF_BOOL, jboolean, Boolean);
CASE(TF_UINT8, jbyte, Byte);
#undef CASE
default:
throwException(env, kIllegalStateException, "invalid DataType(%d)",
dtype);
}
return 0;
}
size_t writeNDArray(JNIEnv* env, jarray src, TF_DataType dtype, int dims_left,
char* dst, size_t dst_size) {
if (dims_left == 1) {
return write1DArray(env, src, dtype, dst, dst_size);
} else {
jobjectArray ndarray = static_cast<jobjectArray>(src);
int len = env->GetArrayLength(ndarray);
size_t sz = 0;
for (int i = 0; i < len; ++i) {
jarray row = static_cast<jarray>(env->GetObjectArrayElement(ndarray, i));
sz +=
writeNDArray(env, row, dtype, dims_left - 1, dst + sz, dst_size - sz);
env->DeleteLocalRef(row);
if (env->ExceptionCheck()) return sz;
}
return sz;
}
}
size_t readNDArray(JNIEnv* env, TF_DataType dtype, const char* src,
size_t src_size, int dims_left, jarray dst) {
if (dims_left == 1) {
return read1DArray(env, dtype, src, src_size, dst);
} else {
jobjectArray ndarray = static_cast<jobjectArray>(dst);
int len = env->GetArrayLength(ndarray);
size_t sz = 0;
for (int i = 0; i < len; ++i) {
jarray row = static_cast<jarray>(env->GetObjectArrayElement(ndarray, i));
sz +=
readNDArray(env, dtype, src + sz, src_size - sz, dims_left - 1, row);
env->DeleteLocalRef(row);
if (env->ExceptionCheck()) return sz;
}
return sz;
}
}
jbyteArray TF_StringDecodeTojbyteArray(JNIEnv* env, const TF_TString* src) {
const char* dst = TF_TString_GetDataPointer(src);
size_t dst_len = TF_TString_GetSize(src);
jbyteArray ret = env->NewByteArray(dst_len);
jbyte* cpy = env->GetByteArrayElements(ret, nullptr);
memcpy(cpy, dst, dst_len);
env->ReleaseByteArrayElements(ret, cpy, 0);
return ret;
}
class StringTensorWriter {
public:
StringTensorWriter(TF_Tensor* t, int num_elements)
: index_(0), data_(static_cast<TF_TString*>(TF_TensorData(t))) {}
void Add(const char* src, size_t len, TF_Status* status) {
if (TF_GetCode(status) != TF_OK) return;
TF_TString_Init(&data_[index_]);
TF_TString_Copy(&data_[index_++], src, len);
}
private:
int index_;
TF_TString* data_;
};
class StringTensorReader {
public:
StringTensorReader(const TF_Tensor* t, int num_elements)
: index_(0), data_(static_cast<const TF_TString*>(TF_TensorData(t))) {}
jbyteArray Next(JNIEnv* env, TF_Status* status) {
if (TF_GetCode(status) != TF_OK) return nullptr;
return TF_StringDecodeTojbyteArray(env, &data_[index_++]);
}
private:
int index_;
const TF_TString* data_;
};
void readNDStringArray(JNIEnv* env, StringTensorReader* reader, int dims_left,
jobjectArray dst, TF_Status* status) {
jsize len = env->GetArrayLength(dst);
if (dims_left == 1) {
for (jsize i = 0; i < len; ++i) {
jbyteArray elem = reader->Next(env, status);
if (TF_GetCode(status) != TF_OK) return;
env->SetObjectArrayElement(dst, i, elem);
env->DeleteLocalRef(elem);
}
return;
}
for (jsize i = 0; i < len; ++i) {
jobjectArray arr =
static_cast<jobjectArray>(env->GetObjectArrayElement(dst, i));
readNDStringArray(env, reader, dims_left - 1, arr, status);
env->DeleteLocalRef(arr);
if (TF_GetCode(status) != TF_OK) return;
}
}
} // namespace
JNIEXPORT jlong JNICALL Java_org_tensorflow_Tensor_allocate(JNIEnv* env,
jclass clazz,
jint dtype,
jlongArray shape,
jlong sizeInBytes) {
int num_dims = static_cast<int>(env->GetArrayLength(shape));
jlong* dims = nullptr;
if (num_dims > 0) {
jboolean is_copy;
dims = env->GetLongArrayElements(shape, &is_copy);
}
static_assert(sizeof(jlong) == sizeof(int64_t),
"Java long is not compatible with the TensorFlow C API");
// On some platforms "jlong" is a "long" while "int64_t" is a "long long".
//
// Thus, static_cast<int64_t*>(dims) will trigger a compiler error:
// static_cast from 'jlong *' (aka 'long *') to 'int64_t *' (aka 'long long
// *') is not allowed
//
// Since this array is typically very small, use the guaranteed safe scheme of
// creating a copy.
int64_t* dims_copy = new int64_t[num_dims];
for (int i = 0; i < num_dims; ++i) {
dims_copy[i] = static_cast<int64_t>(dims[i]);
}
TF_Tensor* t = TF_AllocateTensor(static_cast<TF_DataType>(dtype), dims_copy,
num_dims, static_cast<size_t>(sizeInBytes));
delete[] dims_copy;
if (dims != nullptr) {
env->ReleaseLongArrayElements(shape, dims, JNI_ABORT);
}
if (t == nullptr) {
throwException(env, kNullPointerException,
"unable to allocate memory for the Tensor");
return 0;
}
return reinterpret_cast<jlong>(t);
}
JNIEXPORT jlong JNICALL Java_org_tensorflow_Tensor_allocateScalarBytes(
JNIEnv* env, jclass clazz, jbyteArray value) {
// TF_STRING tensors are encoded with a table of 8-byte offsets followed by
// TF_StringEncode-encoded bytes.
size_t src_len = static_cast<int>(env->GetArrayLength(value));
TF_Tensor* t = TF_AllocateTensor(TF_STRING, nullptr, 0, sizeof(TF_TString));
TF_TString* dst = static_cast<TF_TString*>(TF_TensorData(t));
TF_Status* status = TF_NewStatus();
jbyte* jsrc = env->GetByteArrayElements(value, nullptr);
// jsrc is an unsigned byte*, TF_StringEncode requires a char*.
// reinterpret_cast<> for this conversion should be safe.
TF_TString_Init(&dst[0]);
TF_TString_Copy(&dst[0], reinterpret_cast<const char*>(jsrc), src_len);
env->ReleaseByteArrayElements(value, jsrc, JNI_ABORT);
if (!throwExceptionIfNotOK(env, status)) {
TF_DeleteStatus(status);
return 0;
}
TF_DeleteStatus(status);
return reinterpret_cast<jlong>(t);
}
namespace {
void checkForNullEntries(JNIEnv* env, jarray value, int num_dims) {
jsize len = env->GetArrayLength(value);
for (jsize i = 0; i < len; ++i) {
jarray elem = static_cast<jarray>(
env->GetObjectArrayElement(static_cast<jobjectArray>(value), i));
if (elem == nullptr) {
throwException(env, kNullPointerException,
"null entries in provided array");
return;
}
env->DeleteLocalRef(elem);
if (env->ExceptionCheck()) return;
}
}
void fillNonScalarTF_STRINGTensorData(JNIEnv* env, jarray value, int num_dims,
StringTensorWriter* writer,
TF_Status* status) {
if (num_dims == 0) {
jbyte* jsrc =
env->GetByteArrayElements(static_cast<jbyteArray>(value), nullptr);
writer->Add(reinterpret_cast<const char*>(jsrc), env->GetArrayLength(value),
status);
env->ReleaseByteArrayElements(static_cast<jbyteArray>(value), jsrc,
JNI_ABORT);
return;
}
jsize len = env->GetArrayLength(value);
for (jsize i = 0; i < len; ++i) {
jarray elem = static_cast<jarray>(
env->GetObjectArrayElement(static_cast<jobjectArray>(value), i));
fillNonScalarTF_STRINGTensorData(env, elem, num_dims - 1, writer, status);
env->DeleteLocalRef(elem);
if (TF_GetCode(status) != TF_OK) return;
}
}
} // namespace
JNIEXPORT jlong JNICALL Java_org_tensorflow_Tensor_allocateNonScalarBytes(
JNIEnv* env, jclass clazz, jlongArray shape, jobjectArray value) {
// TF_STRING tensors are encoded with a table of 8-byte offsets following by
// TF_StringEncode-encoded bytes.
const int num_dims = static_cast<int>(env->GetArrayLength(shape));
int64_t* dims = new int64_t[num_dims];
int64_t num_elements = 1;
{
jlong* jdims = env->GetLongArrayElements(shape, nullptr);
for (int i = 0; i < num_dims; ++i) {
dims[i] = static_cast<int64_t>(jdims[i]);
num_elements *= dims[i];
}
env->ReleaseLongArrayElements(shape, jdims, JNI_ABORT);
}
checkForNullEntries(env, value, num_dims);
if (env->ExceptionCheck()) return 0;
TF_Tensor* t = TF_AllocateTensor(TF_STRING, dims, num_dims,
sizeof(TF_TString) * num_elements);
if (t == nullptr) {
delete[] dims;
throwException(env, kNullPointerException,
"unable to allocate memory for the Tensor");
return 0;
}
TF_Status* status = TF_NewStatus();
StringTensorWriter writer(t, num_elements);
fillNonScalarTF_STRINGTensorData(env, value, num_dims, &writer, status);
delete[] dims;
jlong ret = 0;
if (!throwExceptionIfNotOK(env, status)) {
TF_DeleteTensor(t);
} else {
ret = reinterpret_cast<jlong>(t);
}
TF_DeleteStatus(status);
return ret;
}
JNIEXPORT void JNICALL Java_org_tensorflow_Tensor_delete(JNIEnv* env,
jclass clazz,
jlong handle) {
if (handle == 0) return;
TF_DeleteTensor(reinterpret_cast<TF_Tensor*>(handle));
}
JNIEXPORT jobject JNICALL Java_org_tensorflow_Tensor_buffer(JNIEnv* env,
jclass clazz,
jlong handle) {
TF_Tensor* t = requireHandle(env, handle);
if (t == nullptr) return nullptr;
void* data = TF_TensorData(t);
const size_t sz = TF_TensorByteSize(t);
return env->NewDirectByteBuffer(data, static_cast<jlong>(sz));
}
JNIEXPORT jint JNICALL Java_org_tensorflow_Tensor_dtype(JNIEnv* env,
jclass clazz,
jlong handle) {
static_assert(sizeof(jint) >= sizeof(TF_DataType),
"TF_DataType in C cannot be represented as an int in Java");
TF_Tensor* t = requireHandle(env, handle);
if (t == nullptr) return 0;
return static_cast<jint>(TF_TensorType(t));
}
JNIEXPORT jlongArray JNICALL Java_org_tensorflow_Tensor_shape(JNIEnv* env,
jclass clazz,
jlong handle) {
TF_Tensor* t = requireHandle(env, handle);
if (t == nullptr) return nullptr;
static_assert(sizeof(jlong) == sizeof(int64_t),
"Java long is not compatible with the TensorFlow C API");
const jsize num_dims = TF_NumDims(t);
jlongArray ret = env->NewLongArray(num_dims);
jlong* dims = env->GetLongArrayElements(ret, nullptr);
for (int i = 0; i < num_dims; ++i) {
dims[i] = static_cast<jlong>(TF_Dim(t, i));
}
env->ReleaseLongArrayElements(ret, dims, 0);
return ret;
}
JNIEXPORT void JNICALL Java_org_tensorflow_Tensor_setValue(JNIEnv* env,
jclass clazz,
jlong handle,
jobject value) {
TF_Tensor* t = requireHandle(env, handle);
if (t == nullptr) return;
int num_dims = TF_NumDims(t);
TF_DataType dtype = TF_TensorType(t);
void* data = TF_TensorData(t);
const size_t sz = TF_TensorByteSize(t);
if (num_dims == 0) {
writeScalar(env, value, dtype, data, sz);
} else {
writeNDArray(env, static_cast<jarray>(value), dtype, num_dims,
static_cast<char*>(data), sz);
}
}
#define DEFINE_GET_SCALAR_METHOD(jtype, dtype, method_suffix) \
JNIEXPORT jtype JNICALL Java_org_tensorflow_Tensor_scalar##method_suffix( \
JNIEnv* env, jclass clazz, jlong handle) { \
jtype ret = 0; \
TF_Tensor* t = requireHandle(env, handle); \
if (t == nullptr) return ret; \
if (TF_NumDims(t) != 0) { \
throwException(env, kIllegalStateException, "Tensor is not a scalar"); \
} else if (TF_TensorType(t) != dtype) { \
throwException(env, kIllegalStateException, "Tensor is not a %s scalar", \
#method_suffix); \
} else { \
memcpy(&ret, TF_TensorData(t), elemByteSize(dtype)); \
} \
return ret; \
}
DEFINE_GET_SCALAR_METHOD(jfloat, TF_FLOAT, Float);
DEFINE_GET_SCALAR_METHOD(jdouble, TF_DOUBLE, Double);
DEFINE_GET_SCALAR_METHOD(jint, TF_INT32, Int);
DEFINE_GET_SCALAR_METHOD(jlong, TF_INT64, Long);
DEFINE_GET_SCALAR_METHOD(jboolean, TF_BOOL, Boolean);
#undef DEFINE_GET_SCALAR_METHOD
JNIEXPORT jbyteArray JNICALL Java_org_tensorflow_Tensor_scalarBytes(
JNIEnv* env, jclass clazz, jlong handle) {
TF_Tensor* t = requireHandle(env, handle);
if (t == nullptr) return nullptr;
if (TF_NumDims(t) != 0) {
throwException(env, kIllegalStateException, "Tensor is not a scalar");
return nullptr;
}
if (TF_TensorType(t) != TF_STRING) {
throwException(env, kIllegalArgumentException,
"Tensor is not a string/bytes scalar");
return nullptr;
}
const TF_TString* data = static_cast<const TF_TString*>(TF_TensorData(t));
jbyteArray ret = TF_StringDecodeTojbyteArray(env, &data[0]);
return ret;
}
JNIEXPORT void JNICALL Java_org_tensorflow_Tensor_readNDArray(JNIEnv* env,
jclass clazz,
jlong handle,
jobject value) {
TF_Tensor* t = requireHandle(env, handle);
if (t == nullptr) return;
int num_dims = TF_NumDims(t);
TF_DataType dtype = TF_TensorType(t);
const void* data = TF_TensorData(t);
const size_t sz = TF_TensorByteSize(t);
if (num_dims == 0) {
throwException(env, kIllegalArgumentException,
"copyTo() is not meant for scalar Tensors, use the scalar "
"accessor (floatValue(), intValue() etc.) instead");
return;
}
if (dtype == TF_STRING) {
int64_t num_elements = 1;
for (int i = 0; i < num_dims; ++i) {
num_elements *= TF_Dim(t, i);
}
StringTensorReader reader(t, num_elements);
TF_Status* status = TF_NewStatus();
readNDStringArray(env, &reader, num_dims, static_cast<jobjectArray>(value),
status);
throwExceptionIfNotOK(env, status);
TF_DeleteStatus(status);
return;
}
readNDArray(env, dtype, static_cast<const char*>(data), sz, num_dims,
static_cast<jarray>(value));
}
tensorflow/java/src/main/native/tensor_jni.h
+156
View File
@@ -0,0 +1,156 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_MAIN_NATIVE_TENSOR_JNI_H_
#define TENSORFLOW_JAVA_SRC_MAIN_NATIVE_TENSOR_JNI_H_
#include <jni.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
* Class: org_tensorflow_Tensor
* Method: allocate
* Signature: (I[JJ)J
*/
JNIEXPORT jlong JNICALL Java_org_tensorflow_Tensor_allocate(JNIEnv *, jclass,
jint, jlongArray,
jlong);
/*
* Class: org_tensorflow_Tensor
* Method: allocateScalarBytes
* Signature: ([B)J
*/
JNIEXPORT jlong JNICALL
Java_org_tensorflow_Tensor_allocateScalarBytes(JNIEnv *, jclass, jbyteArray);
/*
* Class: org_tensorflow_Tensor
* Method: allocateNonScalarBytes
* Signature: ([J[Ljava/lang/Object;)J
*/
JNIEXPORT jlong JNICALL Java_org_tensorflow_Tensor_allocateNonScalarBytes(
JNIEnv *, jclass, jlongArray, jobjectArray);
/*
* Class: org_tensorflow_Tensor
* Method: delete
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_Tensor_delete(JNIEnv *, jclass,
jlong);
/*
* Class: org_tensorflow_Tensor
* Method: buffer
* Signature: (J)Ljava/nio/ByteBuffer;
*/
JNIEXPORT jobject JNICALL Java_org_tensorflow_Tensor_buffer(JNIEnv *, jclass,
jlong);
/*
* Class: org_tensorflow_Tensor
* Method: dtype
* Signature: (J)I
*/
JNIEXPORT jint JNICALL Java_org_tensorflow_Tensor_dtype(JNIEnv *, jclass,
jlong);
/*
* Class: org_tensorflow_Tensor
* Method: shape
* Signature: (J)[J
*/
JNIEXPORT jlongArray JNICALL Java_org_tensorflow_Tensor_shape(JNIEnv *, jclass,
jlong);
/*
* Class: org_tensorflow_Tensor
* Method: setValue
* Signature: (JLjava/lang/Object;)V
*
* REQUIRES: The jobject's type and shape are compatible the with the DataType
* and shape of the Tensor referred to by the jlong handle.
*/
JNIEXPORT void JNICALL Java_org_tensorflow_Tensor_setValue(JNIEnv *, jclass,
jlong, jobject);
/*
* Class: org_tensorflow_Tensor
* Method: scalarFloat
* Signature: (J)F
*
*/
JNIEXPORT jfloat JNICALL Java_org_tensorflow_Tensor_scalarFloat(JNIEnv *,
jclass, jlong);
/*
* Class: org_tensorflow_Tensor
* Method: scalarDouble
* Signature: (J)D
*/
JNIEXPORT jdouble JNICALL Java_org_tensorflow_Tensor_scalarDouble(JNIEnv *,
jclass,
jlong);
/*
* Class: org_tensorflow_Tensor
* Method: scalarInt
* Signature: (J)I
*/
JNIEXPORT jint JNICALL Java_org_tensorflow_Tensor_scalarInt(JNIEnv *, jclass,
jlong);
/*
* Class: org_tensorflow_Tensor
* Method: scalarLong
* Signature: (J)J
*/
JNIEXPORT jlong JNICALL Java_org_tensorflow_Tensor_scalarLong(JNIEnv *, jclass,
jlong);
/*
* Class: org_tensorflow_Tensor
* Method: scalarBoolean
* Signature: (J)Z
*/
JNIEXPORT jboolean JNICALL Java_org_tensorflow_Tensor_scalarBoolean(JNIEnv *,
jclass,
jlong);
/*
* Class: org_tensorflow_Tensor
* Method: scalarBytes
* Signature: (J)[B
*/
JNIEXPORT jbyteArray JNICALL Java_org_tensorflow_Tensor_scalarBytes(JNIEnv *,
jclass,
jlong);
/*
* Class: org_tensorflow_Tensor
* Method: readNDArray
* Signature: (JLjava/lang/Object;)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_Tensor_readNDArray(JNIEnv *, jclass,
jlong, jobject);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif // TENSORFLOW_JAVA_SRC_MAIN_NATIVE_TENSOR_JNI_H_
tensorflow/java/src/main/native/tensorflow_jni.cc
+67
View File
@@ -0,0 +1,67 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/java/src/main/native/tensorflow_jni.h"
#include <limits>
#include "tensorflow/c/c_api.h"
#include "tensorflow/java/src/main/native/exception_jni.h"
JNIEXPORT jstring JNICALL Java_org_tensorflow_TensorFlow_version(JNIEnv* env,
jclass clazz) {
return env->NewStringUTF(TF_Version());
}
JNIEXPORT jbyteArray JNICALL
Java_org_tensorflow_TensorFlow_registeredOpList(JNIEnv* env, jclass clazz) {
TF_Buffer* buf = TF_GetAllOpList();
jint length = static_cast<int>(buf->length);
jbyteArray ret = env->NewByteArray(length);
env->SetByteArrayRegion(ret, 0, length, static_cast<const jbyte*>(buf->data));
TF_DeleteBuffer(buf);
return ret;
}
JNIEXPORT jlong JNICALL Java_org_tensorflow_TensorFlow_libraryLoad(
JNIEnv* env, jclass clazz, jstring filename) {
TF_Status* status = TF_NewStatus();
const char* cname = env->GetStringUTFChars(filename, nullptr);
TF_Library* h = TF_LoadLibrary(cname, status);
throwExceptionIfNotOK(env, status);
env->ReleaseStringUTFChars(filename, cname);
TF_DeleteStatus(status);
return reinterpret_cast<jlong>(h);
}
JNIEXPORT void JNICALL Java_org_tensorflow_TensorFlow_libraryDelete(
JNIEnv* env, jclass clazz, jlong handle) {
if (handle != 0) {
TF_DeleteLibraryHandle(reinterpret_cast<TF_Library*>(handle));
}
}
JNIEXPORT jbyteArray JNICALL Java_org_tensorflow_TensorFlow_libraryOpList(
JNIEnv* env, jclass clazz, jlong handle) {
TF_Buffer buf = TF_GetOpList(reinterpret_cast<TF_Library*>(handle));
if (buf.length > std::numeric_limits<jint>::max()) {
throwException(env, kIndexOutOfBoundsException,
"Serialized OpList is too large for a byte[] array");
return nullptr;
}
auto ret_len = static_cast<jint>(buf.length);
jbyteArray ret = env->NewByteArray(ret_len);
env->SetByteArrayRegion(ret, 0, ret_len, static_cast<const jbyte*>(buf.data));
return ret;
}
tensorflow/java/src/main/native/tensorflow_jni.h
+70
View File
@@ -0,0 +1,70 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_MAIN_NATIVE_TENSORFLOW_JNI_H_
#define TENSORFLOW_JAVA_SRC_MAIN_NATIVE_TENSORFLOW_JNI_H_
#include <jni.h>
#ifdef __cplusplus
extern "C" {
#endif // __cplusplus
/*
* Class: org_tensorflow_TensorFlow
* Method: version
* Signature: ()Ljava/lang/String;
*/
JNIEXPORT jstring JNICALL Java_org_tensorflow_TensorFlow_version(JNIEnv *,
jclass);
/*
* Class: org_tensorflow_TensorFlow
* Method: registeredOpList
* Signature: ()[B
*/
JNIEXPORT jbyteArray JNICALL
Java_org_tensorflow_TensorFlow_registeredOpList(JNIEnv *, jclass);
/*
* Class: org_tensorflow_TensorFlow
* Method: libraryLoad
* Signature: (Ljava/lang/String;)J
*/
JNIEXPORT jlong JNICALL Java_org_tensorflow_TensorFlow_libraryLoad(JNIEnv *,
jclass,
jstring);
/*
* Class: org_tensorflow_TensorFlow
* Method: libraryDelete
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_org_tensorflow_TensorFlow_libraryDelete(JNIEnv *,
jclass,
jlong);
/*
* Class: org_tensorflow_TensorFlow
* Method: libraryOpList
* Signature: (J)[B
*/
JNIEXPORT jbyteArray JNICALL
Java_org_tensorflow_TensorFlow_libraryOpList(JNIEnv *, jclass, jlong);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif // TENSORFLOW_JAVA_SRC_MAIN_NATIVE_TENSORFLOW_JNI_H_
tensorflow/java/src/main/native/utils_jni.cc
+53
View File
@@ -0,0 +1,53 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/java/src/main/native/utils_jni.h"
#include "tensorflow/java/src/main/native/exception_jni.h"
void resolveOutputs(JNIEnv* env, const char* type, jlongArray src_op,
jintArray src_index, TF_Output* dst, jint n) {
if (env->ExceptionCheck()) return;
jint len = env->GetArrayLength(src_op);
if (len != n) {
throwException(env, kIllegalArgumentException,
"expected %d, got %d %s Operations", n, len, type);
return;
}
len = env->GetArrayLength(src_index);
if (len != n) {
throwException(env, kIllegalArgumentException,
"expected %d, got %d %s Operation output indices", n, len,
type);
return;
}
jlong* op_handles = env->GetLongArrayElements(src_op, nullptr);
jint* indices = env->GetIntArrayElements(src_index, nullptr);
for (int i = 0; i < n; ++i) {
if (op_handles[i] == 0) {
throwException(env, kNullPointerException, "invalid %s (#%d of %d)", type,
i, n);
break;
}
dst[i] = TF_Output{reinterpret_cast<TF_Operation*>(op_handles[i]),
static_cast<int>(indices[i])};
}
env->ReleaseIntArrayElements(src_index, indices, JNI_ABORT);
env->ReleaseLongArrayElements(src_op, op_handles, JNI_ABORT);
}
tensorflow/java/src/main/native/utils_jni.h
+33
View File
@@ -0,0 +1,33 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_JAVA_SRC_MAIN_NATIVE_UTILS_JNI_H_
#define TENSORFLOW_JAVA_SRC_MAIN_NATIVE_UTILS_JNI_H_
#include <jni.h>
#include "tensorflow/c/c_api.h"
#ifdef __cplusplus
extern "C" {
#endif // __cplusplus
void resolveOutputs(JNIEnv* env, const char* type, jlongArray src_op,
jintArray src_index, TF_Output* dst, jint n);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif // TENSORFLOW_JAVA_SRC_MAIN_NATIVE_UTILS_JNI_H_
tensorflow/java/src/test/java/org/tensorflow/EagerOperationBuilderTest.java
+145
View File
@@ -0,0 +1,145 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import static org.junit.Assert.fail;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link EagerOperationBuilder} class. */
@RunWith(JUnit4.class)
public class EagerOperationBuilderTest {
@Test
public void failToCreateIfSessionIsClosed() {
EagerSession session = EagerSession.create();
session.close();
try {
new EagerOperationBuilder(session, "Add", "add");
fail();
} catch (IllegalStateException e) {
// expected
}
}
@Test
public void failToBuildOpIfSessionIsClosed() {
EagerOperationBuilder opBuilder;
try (EagerSession session = EagerSession.create()) {
opBuilder = new EagerOperationBuilder(session, "Empty", "empty");
}
try {
opBuilder.setAttr("dtype", DataType.FLOAT);
fail();
} catch (IllegalStateException e) {
// expected
}
}
@Test
public void addInputs() {
try (EagerSession session = EagerSession.create()) {
Operation asrt =
opBuilder(session, "Assert", "assert")
.addInput(TestUtil.constant(session, "Cond", true))
.addInputList(new Output<?>[] {TestUtil.constant(session, "Error", -1)})
.build();
try {
opBuilder(session, "Const", "var").addControlInput(asrt);
fail();
} catch (UnsupportedOperationException e) {
// expected
}
}
}
@Test
public void setDevice() {
try (EagerSession session = EagerSession.create()) {
opBuilder(session, "Add", "SetDevice")
.setDevice("/job:localhost/replica:0/task:0/device:CPU:0")
.addInput(TestUtil.constant(session, "Const1", 2))
.addInput(TestUtil.constant(session, "Const2", 4))
.build();
}
}
@Test
public void setAttrs() {
// The effect of setting an attribute may not easily be visible from the other parts of this
// package's API. Thus, for now, the test simply executes the various setAttr variants to see
// that there are no exceptions.
//
// This is a bit of an awkward test since it has to find operations with attributes of specific
// types that aren't inferred from the input arguments.
try (EagerSession session = EagerSession.create()) {
// dtype, tensor attributes.
try (Tensor<Integer> t = Tensors.create(1)) {
opBuilder(session, "Const", "DataTypeAndTensor")
.setAttr("dtype", DataType.INT32)
.setAttr("value", t)
.build();
}
// type, int (TF "int" attributes are 64-bit signed, so a Java long).
opBuilder(session, "RandomUniform", "DataTypeAndInt")
.addInput(TestUtil.constant(session, "RandomUniformShape", new int[] {1}))
.setAttr("seed", 10)
.setAttr("dtype", DataType.FLOAT)
.build();
// list(int), string
opBuilder(session, "MaxPool", "IntListAndString")
.addInput(TestUtil.constant(session, "MaxPoolInput", new float[2][2][2][2]))
.setAttr("ksize", new long[] {1, 1, 1, 1})
.setAttr("strides", new long[] {1, 1, 1, 1})
.setAttr("padding", "SAME")
.build();
// list(float), device
opBuilder(session, "FractionalMaxPool", "FloatList")
.addInput(TestUtil.constant(session, "FractionalMaxPoolInput", new float[2][2][2][2]))
.setAttr("pooling_ratio", new float[] {1.0f, 1.44f, 1.73f, 1.0f})
.build();
// shape
opBuilder(session, "EnsureShape", "ShapeAttr")
.addInput(TestUtil.constant(session, "Const", new int[2][2]))
.setAttr("shape", Shape.make(2, 2))
.build();
// list(shape)
opBuilder(session, "FIFOQueue", "queue")
.setAttr("component_types", new DataType[] {DataType.INT32, DataType.INT32})
.setAttr("shapes", new Shape[] {Shape.make(2, 2), Shape.make(2, 2, 2)})
.build();
// bool
opBuilder(session, "All", "Bool")
.addInput(TestUtil.constant(session, "Const", new boolean[] {true, true, false}))
.addInput(TestUtil.constant(session, "Axis", 0))
.setAttr("keep_dims", false)
.build();
// float
opBuilder(session, "ApproximateEqual", "Float")
.addInput(TestUtil.constant(session, "Const1", 10.00001f))
.addInput(TestUtil.constant(session, "Const2", 10.00000f))
.setAttr("tolerance", 0.1f)
.build();
// Missing tests: list(string), list(byte), list(bool), list(type)
}
}
private static EagerOperationBuilder opBuilder(EagerSession session, String type, String name) {
return new EagerOperationBuilder(session, type, name);
}
}
tensorflow/java/src/test/java/org/tensorflow/EagerOperationTest.java
+180
View File
@@ -0,0 +1,180 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link EagerOperation} class. */
@RunWith(JUnit4.class)
public class EagerOperationTest {
@Test
public void failToCreateIfSessionIsClosed() {
EagerSession session = EagerSession.create();
session.close();
try {
new EagerOperation(session, 1L, new long[] {1L}, "Add", "add");
fail();
} catch (IllegalStateException e) {
// expected
}
}
@Test
public void outputDataTypeAndShape() {
try (EagerSession session = EagerSession.create();
Tensor<Integer> t = Tensors.create(new int[2][3])) {
EagerOperation op =
opBuilder(session, "Const", "OutputAttrs")
.setAttr("dtype", DataType.INT32)
.setAttr("value", t)
.build();
assertEquals(DataType.INT32, op.dtype(0));
assertEquals(2, op.shape(0)[0]);
assertEquals(3, op.shape(0)[1]);
}
}
@Test
public void outputTensor() {
try (EagerSession session = EagerSession.create()) {
EagerOperation add =
opBuilder(session, "Add", "CompareResult")
.addInput(TestUtil.constant(session, "Const1", 2))
.addInput(TestUtil.constant(session, "Const2", 4))
.build();
assertEquals(6, add.tensor(0).intValue());
// Validate that we retrieve the right shape and datatype from the tensor
// that has been resolved
assertEquals(0, add.shape(0).length);
assertEquals(DataType.INT32, add.dtype(0));
}
}
@Test
public void inputAndOutputListLengths() {
try (EagerSession session = EagerSession.create()) {
Output<Float> c1 = TestUtil.constant(session, "Const1", new float[] {1f, 2f});
Output<Float> c2 = TestUtil.constant(session, "Const2", new float[] {3f, 4f});
EagerOperation acc =
opBuilder(session, "AddN", "InputListLength")
.addInputList(new Output<?>[] {c1, c2})
.build();
assertEquals(2, acc.inputListLength("inputs"));
assertEquals(1, acc.outputListLength("sum"));
EagerOperation split =
opBuilder(session, "Split", "OutputListLength")
.addInput(TestUtil.constant(session, "Axis", 0))
.addInput(c1)
.setAttr("num_split", 2)
.build();
assertEquals(1, split.inputListLength("split_dim"));
assertEquals(2, split.outputListLength("output"));
try {
split.inputListLength("no_such_input");
fail();
} catch (IllegalArgumentException e) {
// expected
}
try {
split.outputListLength("no_such_output");
fail();
} catch (IllegalArgumentException e) {
// expected
}
}
}
@Test
public void numOutputs() {
try (EagerSession session = EagerSession.create()) {
EagerOperation op =
opBuilder(session, "UniqueWithCountsV2", "unq")
.addInput(TestUtil.constant(session, "Const1", new int[] {1, 2, 1}))
.addInput(TestUtil.constant(session, "Axis", new int[] {0}))
.setAttr("out_idx", DataType.INT32)
.build();
assertEquals(3, op.numOutputs());
}
}
@Test
public void opNotAccessibleIfSessionIsClosed() {
EagerSession session = EagerSession.create();
EagerOperation add =
opBuilder(session, "Add", "SessionClosed")
.addInput(TestUtil.constant(session, "Const1", 2))
.addInput(TestUtil.constant(session, "Const2", 4))
.build();
assertEquals(1, add.outputListLength("z"));
session.close();
try {
add.outputListLength("z");
fail();
} catch (IllegalStateException e) {
// expected
}
}
@Test
public void outputIndexOutOfBounds() {
try (EagerSession session = EagerSession.create()) {
EagerOperation add =
opBuilder(session, "Add", "OutOfRange")
.addInput(TestUtil.constant(session, "Const1", 2))
.addInput(TestUtil.constant(session, "Const2", 4))
.build();
try {
add.getUnsafeNativeHandle(1);
fail();
} catch (IndexOutOfBoundsException e) {
// expected
}
try {
add.shape(1);
fail();
} catch (IndexOutOfBoundsException e) {
// expected
}
try {
add.dtype(1);
fail();
} catch (IndexOutOfBoundsException e) {
// expected
}
try {
add.tensor(1);
fail();
} catch (IndexOutOfBoundsException e) {
// expected
}
}
}
private static EagerOperationBuilder opBuilder(EagerSession session, String type, String name) {
return new EagerOperationBuilder(session, type, name);
}
}
tensorflow/java/src/test/java/org/tensorflow/EagerSessionTest.java
+221
View File
@@ -0,0 +1,221 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.lang.ref.Reference;
import java.lang.ref.ReferenceQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.tensorflow.EagerSession.ResourceCleanupStrategy;
@RunWith(JUnit4.class)
public class EagerSessionTest {
@Test
public void closeSessionTwiceDoesNotFail() {
try (EagerSession s = EagerSession.create()) {
s.close();
}
}
@Test
public void cleanupResourceOnSessionClose() {
TestReference ref;
try (EagerSession s =
EagerSession.options()
.resourceCleanupStrategy(ResourceCleanupStrategy.ON_SESSION_CLOSE)
.build()) {
ref = new TestReference(s, new Object());
assertFalse(ref.isDeleted());
// check that reaching safe point did not release resources
buildOp(s);
assertFalse(ref.isDeleted());
}
assertTrue(ref.isDeleted());
}
@Test
public void cleanupResourceOnSafePoints() {
TestGarbageCollectorQueue gcQueue = new TestGarbageCollectorQueue();
try (EagerSession s =
EagerSession.options()
.resourceCleanupStrategy(ResourceCleanupStrategy.ON_SAFE_POINTS)
.buildForGcTest(gcQueue)) {
TestReference ref = new TestReference(s, new Object());
assertFalse(ref.isDeleted());
// garbage collecting the reference won't release until we reached safe point
gcQueue.collect(ref);
assertFalse(ref.isDeleted());
buildOp(s); // safe point
assertTrue(ref.isDeleted());
assertTrue(gcQueue.isEmpty());
}
}
@Test
public void cleanupResourceInBackground() {
TestGarbageCollectorQueue gcQueue = new TestGarbageCollectorQueue();
try (EagerSession s =
EagerSession.options()
.resourceCleanupStrategy(ResourceCleanupStrategy.IN_BACKGROUND)
.buildForGcTest(gcQueue)) {
TestReference ref = new TestReference(s, new Object());
assertFalse(ref.isDeleted());
gcQueue.collect(ref);
sleep(50); // allow some time to the background thread for cleaning up resources
assertTrue(ref.isDeleted());
assertTrue(gcQueue.isEmpty());
}
}
@Test
public void clearedResourcesAreNotCleanedUp() {
TestReference ref;
try (EagerSession s = EagerSession.create()) {
ref = new TestReference(s, new Object());
ref.clear();
}
assertFalse(ref.isDeleted());
}
@Test
public void buildingOpWithClosedSessionFails() {
EagerSession s = EagerSession.create();
s.close();
try {
buildOp(s);
fail();
} catch (IllegalStateException e) {
// ok
}
}
@Test
public void addingReferenceToClosedSessionFails() {
EagerSession s = EagerSession.create();
s.close();
try {
new TestReference(s, new Object());
fail();
} catch (IllegalStateException e) {
// ok
}
}
@Test
public void defaultSession() throws Exception {
EagerSession.Options options =
EagerSession.options().resourceCleanupStrategy(ResourceCleanupStrategy.ON_SESSION_CLOSE);
EagerSession.initDefault(options);
EagerSession session = EagerSession.getDefault();
assertNotNull(session);
assertEquals(ResourceCleanupStrategy.ON_SESSION_CLOSE, session.resourceCleanupStrategy());
try {
EagerSession.initDefault(options);
fail();
} catch (IllegalStateException e) {
// expected
}
try {
session.close();
fail();
} catch (IllegalStateException e) {
// expected
}
}
private static class TestReference extends EagerSession.NativeReference {
TestReference(EagerSession session, Object referent) {
super(session, referent);
}
@Override
void delete() {
if (!deleted.compareAndSet(false, true)) {
fail("Reference was deleted more than once");
}
}
boolean isDeleted() {
return deleted.get();
}
private final AtomicBoolean deleted = new AtomicBoolean();
}
private static class TestGarbageCollectorQueue extends ReferenceQueue<Object> {
@Override
public Reference<? extends Object> poll() {
return garbage.poll();
}
@Override
public Reference<? extends Object> remove() throws InterruptedException {
return garbage.take();
}
@Override
public Reference<? extends Object> remove(long timeout)
throws IllegalArgumentException, InterruptedException {
return garbage.poll(timeout, TimeUnit.MILLISECONDS);
}
void collect(TestReference ref) {
garbage.add(ref);
}
boolean isEmpty() {
return garbage.isEmpty();
}
private final BlockingQueue<TestReference> garbage = new LinkedBlockingQueue<>();
}
private static void buildOp(EagerSession s) {
// Creating an operation is a safe point for resource cleanup
try {
s.opBuilder("Const", "Const");
} catch (UnsupportedOperationException e) {
// TODO (karlllessard) remove this exception catch when EagerOperationBuilder is implemented
}
}
private static void sleep(int millis) {
try {
Thread.sleep(millis);
} catch (InterruptedException e) {
}
}
}
tensorflow/java/src/test/java/org/tensorflow/GraphOperationBuilderTest.java
+219
View File
@@ -0,0 +1,219 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link org.tensorflow.GraphOperationBuilder}. */
@RunWith(JUnit4.class)
public class GraphOperationBuilderTest {
// TODO(ashankar): Restore this test once the C API gracefully handles mixing graphs and
// operations instead of segfaulting.
@Test
@Ignore
public void failWhenMixingOperationsOnDifferentGraphs() {
try (Graph g1 = new Graph();
Graph g2 = new Graph()) {
Output<Integer> c1 = TestUtil.constant(g1, "C1", 3);
Output<Integer> c2 = TestUtil.constant(g2, "C2", 3);
TestUtil.addN(g1, c1, c1);
try {
TestUtil.addN(g2, c1, c2);
} catch (Exception e) {
fail(e.toString());
}
}
}
@Test
public void failOnUseAfterBuild() {
try (Graph g = new Graph();
Tensor<Integer> t = Tensors.create(1)) {
OperationBuilder b =
g.opBuilder("Const", "Const").setAttr("dtype", t.dataType()).setAttr("value", t);
b.build();
try {
b.setAttr("dtype", t.dataType());
} catch (IllegalStateException e) {
// expected exception.
}
}
}
@Test
public void failOnUseAfterGraphClose() {
OperationBuilder b = null;
try (Graph g = new Graph();
Tensor<Integer> t = Tensors.create(1)) {
b = g.opBuilder("Const", "Const").setAttr("dtype", t.dataType()).setAttr("value", t);
}
try {
b.build();
} catch (IllegalStateException e) {
// expected exception.
}
}
@Test
public void setAttr() {
// The effect of setting an attribute may not easily be visible from the other parts of this
// package's API. Thus, for now, the test simply executes the various setAttr variants to see
// that there are no exceptions. If an attribute is "visible", test for that in a separate test
// (like setAttrShape).
//
// This is a bit of an awkward test since it has to find operations with attributes of specific
// types that aren't inferred from the input arguments.
try (Graph g = new Graph()) {
// dtype, tensor attributes.
try (Tensor<Integer> t = Tensors.create(1)) {
g.opBuilder("Const", "DataTypeAndTensor")
.setAttr("dtype", DataType.INT32)
.setAttr("value", t)
.build()
.output(0);
assertTrue(hasNode(g, "DataTypeAndTensor"));
}
// string, bool attributes.
g.opBuilder("Abort", "StringAndBool")
.setAttr("error_msg", "SomeErrorMessage")
.setAttr("exit_without_error", false)
.build();
assertTrue(hasNode(g, "StringAndBool"));
// int (TF "int" attributes are 64-bit signed, so a Java long).
g.opBuilder("RandomUniform", "Int")
.addInput(TestUtil.constant(g, "RandomUniformShape", new int[] {1}))
.setAttr("seed", 10)
.setAttr("dtype", DataType.FLOAT)
.build();
assertTrue(hasNode(g, "Int"));
// list(int)
g.opBuilder("MaxPool", "IntList")
.addInput(TestUtil.constant(g, "MaxPoolInput", new float[2][2][2][2]))
.setAttr("ksize", new long[] {1, 1, 1, 1})
.setAttr("strides", new long[] {1, 1, 1, 1})
.setAttr("padding", "SAME")
.build();
assertTrue(hasNode(g, "IntList"));
// list(float)
g.opBuilder("FractionalMaxPool", "FloatList")
.addInput(TestUtil.constant(g, "FractionalMaxPoolInput", new float[2][2][2][2]))
.setAttr("pooling_ratio", new float[] {1.0f, 1.44f, 1.73f, 1.0f})
.build();
assertTrue(hasNode(g, "FloatList"));
// Missing tests: float, list(dtype), list(tensor), list(string), list(bool)
}
}
@Test
public void setAttrShape() {
try (Graph g = new Graph()) {
Output<?> n =
g.opBuilder("Placeholder", "unknown")
.setAttr("dtype", DataType.FLOAT)
.setAttr("shape", Shape.unknown())
.build()
.output(0);
assertEquals(-1, n.shape().numDimensions());
assertEquals(DataType.FLOAT, n.dataType());
n =
g.opBuilder("Placeholder", "batch_of_vectors")
.setAttr("dtype", DataType.FLOAT)
.setAttr("shape", Shape.make(-1, 784))
.build()
.output(0);
assertEquals(2, n.shape().numDimensions());
assertEquals(-1, n.shape().size(0));
assertEquals(784, n.shape().size(1));
assertEquals(DataType.FLOAT, n.dataType());
}
}
@Test
public void setAttrShapeList() {
// Those shapes match tensors ones, so no exception is thrown
testSetAttrShapeList(new Shape[] {Shape.make(2, 2), Shape.make(2, 2, 2)});
try {
// Those shapes do not match tensors ones, exception is thrown
testSetAttrShapeList(new Shape[] {Shape.make(2, 2), Shape.make(2, 2, 2, 2)});
fail("Shapes are incompatible and an exception was expected");
} catch (IllegalArgumentException e) {
// expected
}
}
@Test
public void addControlInput() {
try (Graph g = new Graph();
Session s = new Session(g);
Tensor<Boolean> yes = Tensors.create(true);
Tensor<Boolean> no = Tensors.create(false)) {
Output<Boolean> placeholder = TestUtil.placeholder(g, "boolean", Boolean.class);
GraphOperation check =
g.opBuilder("Assert", "assert")
.addInput(placeholder)
.addInputList(new Output<?>[] {placeholder})
.build();
Operation noop = g.opBuilder("NoOp", "noop").addControlInput(check).build();
// No problems when the Assert check succeeds
s.runner().feed(placeholder, yes).addTarget(noop).run();
// Exception thrown by the execution of the Assert node
try {
s.runner().feed(placeholder, no).addTarget(noop).run();
fail("Did not run control operation.");
} catch (IllegalArgumentException e) {
// expected
}
}
}
private static void testSetAttrShapeList(Shape[] shapes) {
try (Graph g = new Graph();
Session s = new Session(g)) {
int[][] matrix = new int[][] {{0, 0}, {0, 0}};
Output<?> queue =
g.opBuilder("FIFOQueue", "queue")
.setAttr("component_types", new DataType[] {DataType.INT32, DataType.INT32})
.setAttr("shapes", shapes)
.build()
.output(0);
assertTrue(hasNode(g, "queue"));
Output<Integer> c1 = TestUtil.constant(g, "const1", matrix);
Output<Integer> c2 = TestUtil.constant(g, "const2", new int[][][] {matrix, matrix});
Operation enqueue =
g.opBuilder("QueueEnqueue", "enqueue")
.addInput(queue)
.addInputList(new Output<?>[] {c1, c2})
.build();
assertTrue(hasNode(g, "enqueue"));
s.runner().addTarget(enqueue).run();
}
}
private static boolean hasNode(Graph g, String name) {
return g.operation(name) != null;
}
}
tensorflow/java/src/test/java/org/tensorflow/GraphOperationTest.java
+203
View File
@@ -0,0 +1,203 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link org.tensorflow.GraphOperation}. */
@RunWith(JUnit4.class)
public class GraphOperationTest {
@Test
public void outputListLengthFailsOnInvalidName() {
try (Graph g = new Graph()) {
Operation op =
g.opBuilder("Add", "Add")
.addInput(TestUtil.constant(g, "x", 1))
.addInput(TestUtil.constant(g, "y", 2))
.build();
assertEquals(1, op.outputListLength("z"));
try {
op.outputListLength("unknown");
fail("Did not catch bad name");
} catch (IllegalArgumentException iae) {
// expected
}
}
}
@Test
public void operationEquality() {
GraphOperation op1;
try (Graph g = new Graph()) {
op1 = TestUtil.constantOp(g, "op1", 1);
GraphOperation op2 = TestUtil.constantOp(g, "op2", 2);
GraphOperation op3 = new GraphOperation(g, op1.getUnsafeNativeHandle());
GraphOperation op4 = g.operation("op1");
assertEquals(op1, op1);
assertNotEquals(op1, op2);
assertEquals(op1, op3);
assertEquals(op1.hashCode(), op3.hashCode());
assertEquals(op1, op4);
assertEquals(op1.hashCode(), op4.hashCode());
assertEquals(op3, op4);
assertNotEquals(op2, op3);
assertNotEquals(op2, op4);
}
try (Graph g = new Graph()) {
Operation newOp1 = TestUtil.constant(g, "op1", 1).op();
assertNotEquals(op1, newOp1);
}
}
@Test
public void operationCollection() {
try (Graph g = new Graph()) {
GraphOperation op1 = TestUtil.constantOp(g, "op1", 1);
GraphOperation op2 = TestUtil.constantOp(g, "op2", 2);
GraphOperation op3 = new GraphOperation(g, op1.getUnsafeNativeHandle());
GraphOperation op4 = g.operation("op1");
Set<Operation> ops = new HashSet<>();
ops.addAll(Arrays.asList(op1, op2, op3, op4));
assertEquals(2, ops.size());
assertTrue(ops.contains(op1));
assertTrue(ops.contains(op2));
assertTrue(ops.contains(op3));
assertTrue(ops.contains(op4));
}
}
@Test
public void operationToString() {
try (Graph g = new Graph()) {
Operation op = TestUtil.constant(g, "c", new int[] {1}).op();
assertNotNull(op.toString());
}
}
@Test
public void outputEquality() {
try (Graph g = new Graph()) {
Output<Integer> output = TestUtil.constant(g, "c", 1);
Output<Integer> output1 = output.op().<Integer>output(0);
Output<Integer> output2 = g.operation("c").<Integer>output(0);
assertEquals(output, output1);
assertEquals(output.hashCode(), output1.hashCode());
assertEquals(output, output2);
assertEquals(output.hashCode(), output2.hashCode());
}
}
@Test
public void outputCollection() {
try (Graph g = new Graph()) {
Output<Integer> output = TestUtil.constant(g, "c", 1);
Output<Integer> output1 = output.op().<Integer>output(0);
Output<Integer> output2 = g.operation("c").<Integer>output(0);
Set<Output<Integer>> ops = new HashSet<>();
ops.addAll(Arrays.asList(output, output1, output2));
assertEquals(1, ops.size());
assertTrue(ops.contains(output));
assertTrue(ops.contains(output1));
assertTrue(ops.contains(output2));
}
}
@Test
public void outputToString() {
try (Graph g = new Graph()) {
Output<Integer> output = TestUtil.constant(g, "c", new int[] {1});
assertNotNull(output.toString());
}
}
@Test
public void outputListLength() {
assertEquals(1, split(new int[] {0, 1}, 1));
assertEquals(2, split(new int[] {0, 1}, 2));
assertEquals(3, split(new int[] {0, 1, 2}, 3));
}
@Test
public void inputListLength() {
assertEquals(1, splitWithInputList(new int[] {0, 1}, 1, "split_dim"));
try {
splitWithInputList(new int[] {0, 1}, 2, "inputs");
} catch (IllegalArgumentException iae) {
// expected
}
}
@Test
public void outputList() {
try (Graph g = new Graph()) {
Operation split = TestUtil.split(g, "split", new int[] {0, 1, 2}, 3);
Output<?>[] outputs = split.outputList(1, 2);
assertNotNull(outputs);
assertEquals(2, outputs.length);
for (int i = 0; i < outputs.length; ++i) {
assertEquals(i + 1, outputs[i].index());
}
}
}
@Test
public void outputTensorNotSupported() {
try (Graph g = new Graph()) {
Operation split = TestUtil.split(g, "split", new int[] {0, 1, 2}, 3);
try {
split.output(0).tensor();
fail();
} catch (IllegalStateException e) {
}
}
}
private static int split(int[] values, int num_split) {
try (Graph g = new Graph()) {
return g.opBuilder("Split", "Split")
.addInput(TestUtil.constant(g, "split_dim", 0))
.addInput(TestUtil.constant(g, "values", values))
.setAttr("num_split", num_split)
.build()
.outputListLength("output");
}
}
private static int splitWithInputList(int[] values, int num_split, String name) {
try (Graph g = new Graph()) {
return g.opBuilder("Split", "Split")
.addInput(TestUtil.constant(g, "split_dim", 0))
.addInput(TestUtil.constant(g, "values", values))
.setAttr("num_split", num_split)
.build()
.inputListLength(name);
}
}
}
tensorflow/java/src/test/java/org/tensorflow/GraphTest.java
+369
View File
@@ -0,0 +1,369 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.util.HashSet;
import java.util.Iterator;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link org.tensorflow.Graph}. */
@RunWith(JUnit4.class)
public class GraphTest {
@Test
public void graphDefRoundTrip() {
byte[] graphDef;
// Create a graph for A * X + B
try (Graph g = new Graph()) {
TestUtil.transpose_A_times_X(g, new int[2][2]);
graphDef = g.toGraphDef();
}
// Import the GraphDef and find all the nodes.
try (Graph g = new Graph()) {
g.importGraphDef(graphDef);
validateImportedGraph(g, "");
}
try (Graph g = new Graph()) {
g.importGraphDef(graphDef, "BugsBunny");
validateImportedGraph(g, "BugsBunny/");
}
}
// Helper function whose implementation is based on knowledge of how
// TestUtil.transpose_A_times_X is implemented.
private static void validateImportedGraph(Graph g, String prefix) {
Operation op = g.operation(prefix + "A");
assertNotNull(op);
assertEquals(prefix + "A", op.name());
assertEquals("Const", op.type());
assertEquals(1, op.numOutputs());
assertEquals(op, op.output(0).op());
op = g.operation(prefix + "X");
assertNotNull(op);
assertEquals(prefix + "X", op.name());
assertEquals("Placeholder", op.type());
assertEquals(1, op.numOutputs());
assertEquals(op, op.output(0).op());
op = g.operation(prefix + "Y");
assertNotNull(op);
assertEquals(prefix + "Y", op.name());
assertEquals("MatMul", op.type());
assertEquals(1, op.numOutputs());
assertEquals(op, op.output(0).op());
}
@Test
public void iterateOverOperations() {
try (Graph g = new Graph()) {
Iterator<Operation> iterator = g.operations();
HashSet<Operation> operations;
assertFalse(iterator.hasNext());
operations = new HashSet<>();
operations.add(TestUtil.constant(g, "Const-A", Float.valueOf(1.0f)).op());
operations.add(TestUtil.constant(g, "Const-B", Integer.valueOf(23)).op());
operations.add(TestUtil.constant(g, "Const-C", Double.valueOf(1.618)).op());
iterator = g.operations();
assertTrue(iterator.hasNext());
assertTrue(operations.remove(iterator.next()));
assertTrue(iterator.hasNext());
assertTrue(operations.remove(iterator.next()));
assertTrue(iterator.hasNext());
assertTrue(operations.remove(iterator.next()));
assertFalse(iterator.hasNext());
}
}
@Test
public void failImportOnInvalidGraphDefs() {
try (Graph g = new Graph()) {
try {
g.importGraphDef(null);
} catch (IllegalArgumentException e) {
// expected exception.
}
try {
g.importGraphDef(new byte[] {1});
} catch (IllegalArgumentException e) {
// expected exception.
}
}
}
@Test
public void failOnUseAfterClose() {
Graph g = new Graph();
g.close();
try {
g.toGraphDef();
} catch (IllegalStateException e) {
// expected exception.
}
}
@Test
public void addGradientsToGraph() {
try (Graph g = new Graph();
Session s = new Session(g)) {
Output<Float> x1 = TestUtil.placeholder(g, "x1", Float.class);
Output<Float> x2 = TestUtil.placeholder(g, "x2", Float.class);
Output<Float> y0 = TestUtil.square(g, "y0", x1);
Output<Float> y1 = TestUtil.square(g, "y1", y0);
Output<Float> y2 = TestUtil.addN(g, y0, x2);
Output<?>[] grads0 = g.addGradients(y1, toArray(x1));
assertNotNull(grads0);
assertEquals(1, grads0.length);
assertEquals(DataType.FLOAT, grads0[0].dataType());
Output<?>[] grads1 = g.addGradients(y2, toArray(x1, x2));
assertNotNull(grads1);
assertEquals(2, grads1.length);
assertEquals(DataType.FLOAT, grads1[0].dataType());
assertEquals(DataType.FLOAT, grads1[1].dataType());
try (Tensor<Float> c1 = Tensors.create(3.0f);
Tensor<Float> c2 = Tensors.create(2.0f);
TestUtil.AutoCloseableList<Tensor<?>> outputs = new TestUtil.AutoCloseableList<>(
s.runner()
.feed(x1, c1)
.feed(x2, c2)
.fetch(grads0[0])
.fetch(grads1[0])
.fetch(grads1[1])
.run())) {
assertEquals(3, outputs.size());
assertEquals(108.0f, outputs.get(0).floatValue(), 0.0f);
assertEquals(6.0f, outputs.get(1).floatValue(), 0.0f);
assertEquals(1.0f, outputs.get(2).floatValue(), 0.0f);
}
}
}
@Test
public void addGradientSumsToGraph() {
try (Graph g = new Graph();
Session s = new Session(g)) {
Output<Float> x = TestUtil.placeholder(g, "x", Float.class);
Output<Float> y0 = TestUtil.square(g, "y0", x);
Output<Float> y1 = TestUtil.square(g, "y1", y0);
Output<?>[] grad = g.addGradients(null, toArray(y0, y1), toArray(x), null);
assertNotNull(grad);
assertEquals(1, grad.length);
assertEquals(DataType.FLOAT, grad[0].dataType());
try (Tensor<Float> c = Tensors.create(3.0f);
Tensor<?> output = s.runner()
.feed(x, c)
.fetch(grad[0])
.run()
.get(0)) {
assertEquals(114.0f, output.floatValue(), 0.0f);
}
}
}
@Test
public void addGradientsWithInitialValuesToGraph() {
try (Graph g = new Graph();
Session s = new Session(g)) {
Output<Float> x = TestUtil.placeholder(g, "x", Float.class);
Output<Float> y0 = TestUtil.square(g, "y0", x);
Output<Float> y1 = TestUtil.square(g, "y1", y0);
Output<?>[] grad0 = g.addGradients(y1, toArray(y0));
assertNotNull(grad0);
assertEquals(1, grad0.length);
assertEquals(DataType.FLOAT, grad0[0].dataType());
Output<?>[] grad1 = g.addGradients(null, toArray(y0), toArray(x), toArray(grad0[0]));
assertNotNull(grad1);
assertEquals(1, grad1.length);
assertEquals(DataType.FLOAT, grad1[0].dataType());
try (Tensor<Float> c = Tensors.create(3.0f);
Tensor<?> output = s.runner()
.feed(x, c)
.fetch(grad1[0])
.run()
.get(0)) {
assertEquals(108.0f, output.floatValue(), 0.0f);
}
}
}
@Test
public void validateGradientsNames() {
try (Graph g = new Graph()) {
Output<Float> x = TestUtil.placeholder(g, "x", Float.class);
Output<Float> y0 = TestUtil.square(g, "y0", x);
Output<?>[] grad0 = g.addGradients(null, toArray(y0), toArray(x), null);
assertTrue(grad0[0].op().name().startsWith("gradients/"));
Output<?>[] grad1 = g.addGradients(null, toArray(y0), toArray(x), null);
assertTrue(grad1[0].op().name().startsWith("gradients_1/"));
Output<?>[] grad2 = g.addGradients("more_gradients", toArray(y0), toArray(x), null);
assertTrue(grad2[0].op().name().startsWith("more_gradients/"));
Output<?>[] grad3 = g.addGradients("even_more_gradients", toArray(y0), toArray(x), null);
assertTrue(grad3[0].op().name().startsWith("even_more_gradients/"));
try {
g.addGradients("even_more_gradients", toArray(y0), toArray(x), null);
} catch (IllegalArgumentException e) {
// expected exception
}
}
}
@Test
public void buildWhileLoopSingleInput() {
try (Graph g = new Graph();
Session s = new Session(g)) {
Output<?> input = TestUtil.placeholder(g, "input1", Integer.class);
// could write this using lambda after Java 8
Graph.WhileSubgraphBuilder condGraphBuilder =
new Graph.WhileSubgraphBuilder() {
@Override
public void buildSubgraph(
Graph condGraph, Output<?>[] condInputs, Output<?>[] condOutputs) {
Output<Integer> sixteen = TestUtil.constant(condGraph, "sixteen", 16);
// condInputs[0] < 16
Output<?> condOutput =
condGraph
.opBuilder("Less", "cond")
.addInput(condInputs[0])
.addInput(sixteen)
.build()
.output(0);
condOutputs[0] = condOutput;
}
};
// could write this using lambda after Java 8
Graph.WhileSubgraphBuilder bodyGraphBuilder =
new Graph.WhileSubgraphBuilder() {
@Override
public void buildSubgraph(
Graph bodyGraph, Output<?>[] bodyInputs, Output<?>[] bodyOutputs) {
bodyOutputs[0] = TestUtil.square(bodyGraph, "square", bodyInputs[0]);
}
};
Output<?>[] loopOutputs =
g.whileLoop(toArray(input), condGraphBuilder, bodyGraphBuilder, "test_loop");
try (Tensor<Integer> c = Tensors.create(2);
Tensor<?> output = s.runner().feed(input, c).fetch(loopOutputs[0]).run().get(0)) {
assertEquals(16, output.intValue()); // ((2^2)^2)
}
}
}
@Test
public void buildWhileLoopMultipleInputs() {
try (Graph g = new Graph();
Session s = new Session(g)) {
Output<?> input1 = TestUtil.placeholder(g, "input1", Integer.class);
Output<?> input2 = TestUtil.placeholder(g, "input2", Integer.class);
Output<?>[] inputs = toArray(input1, input2);
// could write this using lambda after Java 8
Graph.WhileSubgraphBuilder condGraphBuilder =
new Graph.WhileSubgraphBuilder() {
@Override
public void buildSubgraph(
Graph condGraph, Output<?>[] condInputs, Output<?>[] condOutputs) {
Output<Integer> sixteen = TestUtil.constant(condGraph, "sixteen", 16);
Output<?> condOutput =
condGraph
.opBuilder("Less", "cond")
.addInput(condInputs[0])
.addInput(sixteen)
.build()
.output(0); // condInputs[0] < 16
condOutputs[0] = condOutput;
}
};
// could write this using lambda after Java 8
Graph.WhileSubgraphBuilder bodyGraphBuilder =
new Graph.WhileSubgraphBuilder() {
@Override
public void buildSubgraph(
Graph bodyGraph, Output<?>[] bodyInputs, Output<?>[] bodyOutputs) {
bodyOutputs[0] = TestUtil.square(bodyGraph, "square1", bodyInputs[0]);
bodyOutputs[1] = TestUtil.square(bodyGraph, "square2", bodyInputs[1]);
}
};
Output<?>[] loopOutputs =
g.whileLoop(inputs, condGraphBuilder, bodyGraphBuilder, "test_loop");
try (Tensor<Integer> c1 = Tensors.create(2);
Tensor<Integer> c2 = Tensors.create(5);
TestUtil.AutoCloseableList<Tensor<?>> outputs =
new TestUtil.AutoCloseableList<>(
s.runner()
.feed(input1, c1)
.feed(input2, c2)
.fetch(loopOutputs[0])
.fetch(loopOutputs[1])
.run())) {
assertEquals(2, outputs.size());
assertEquals(16, outputs.get(0).intValue()); // ((2^2)^2)
assertEquals(625, outputs.get(1).intValue()); // ((5^2)^2)
}
}
}
private static Output<?>[] toArray(Output<?>... outputs) {
return outputs;
}
}
tensorflow/java/src/test/java/org/tensorflow/SavedModelBundleTest.java
+107
View File
@@ -0,0 +1,107 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link org.tensorflow.SavedModelBundle}. */
@RunWith(JUnit4.class)
public class SavedModelBundleTest {
private static final String SAVED_MODEL_PATH =
"tensorflow/cc/saved_model/testdata/half_plus_two/00000123";
@Test
public void load() {
try (SavedModelBundle bundle = SavedModelBundle.load(SAVED_MODEL_PATH, "serve")) {
assertNotNull(bundle.session());
assertNotNull(bundle.graph());
assertNotNull(bundle.metaGraphDef());
}
}
@Test
public void loadNonExistentBundle() {
try {
SavedModelBundle bundle = SavedModelBundle.load("__BAD__", "serve");
bundle.close();
fail("not expected");
} catch (org.tensorflow.TensorFlowException e) {
// expected exception
assertTrue(e.getMessage().contains("Could not find SavedModel"));
}
}
@Test
public void loader() {
try (SavedModelBundle bundle = SavedModelBundle.loader(SAVED_MODEL_PATH)
.withTags("serve")
.withConfigProto(sillyConfigProto())
.withRunOptions(sillyRunOptions())
.load()) {
assertNotNull(bundle.session());
assertNotNull(bundle.graph());
assertNotNull(bundle.metaGraphDef());
}
}
private static byte[] sillyRunOptions() {
// Ideally this would use the generated Java sources for protocol buffers
// and end up with something like the snippet below. However, generating
// the Java files for the .proto files in tensorflow/core:protos_all is
// a bit cumbersome in bazel until the proto_library rule is setup.
//
// See https://github.com/bazelbuild/bazel/issues/52#issuecomment-194341866
// https://github.com/bazelbuild/rules_go/pull/121#issuecomment-251515362
// https://github.com/bazelbuild/rules_go/pull/121#issuecomment-251692558
//
// For this test, for now, the use of specific bytes suffices.
return new byte[] {0x08, 0x03};
/*
return org.tensorflow.framework.RunOptions.newBuilder()
.setTraceLevel(RunOptions.TraceLevel.FULL_TRACE)
.build()
.toByteArray();
*/
}
public static byte[] sillyConfigProto() {
// Ideally this would use the generated Java sources for protocol buffers
// and end up with something like the snippet below. However, generating
// the Java files for the .proto files in tensorflow/core:protos_all is
// a bit cumbersome in bazel until the proto_library rule is setup.
//
// See https://github.com/bazelbuild/bazel/issues/52#issuecomment-194341866
// https://github.com/bazelbuild/rules_go/pull/121#issuecomment-251515362
// https://github.com/bazelbuild/rules_go/pull/121#issuecomment-251692558
//
// For this test, for now, the use of specific bytes suffices.
return new byte[] {0x10, 0x01, 0x28, 0x01};
/*
return org.tensorflow.framework.ConfigProto.newBuilder()
.setInterOpParallelismThreads(1)
.setIntraOpParallelismThreads(1)
.build()
.toByteArray();
*/
}
}
tensorflow/java/src/test/java/org/tensorflow/SessionTest.java
+205
View File
@@ -0,0 +1,205 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link org.tensorflow.Session}. */
@RunWith(JUnit4.class)
public class SessionTest {
@Test
public void runUsingOperationNames() {
try (Graph g = new Graph();
Session s = new Session(g)) {
TestUtil.transpose_A_times_X(g, new int[][] {{2}, {3}});
try (Tensor<Integer> x = Tensors.create(new int[][] {{5}, {7}});
TestUtil.AutoCloseableList<Tensor<?>> outputs =
new TestUtil.AutoCloseableList<Tensor<?>>(s.runner().feed("X", x).fetch("Y").run())) {
assertEquals(1, outputs.size());
final int[][] expected = {{31}};
assertArrayEquals(expected, outputs.get(0).copyTo(new int[1][1]));
}
}
}
@Test
public void runUsingOperationHandles() {
try (Graph g = new Graph();
Session s = new Session(g)) {
TestUtil.transpose_A_times_X(g, new int[][] {{2}, {3}});
Output<Integer> feed = g.operation("X").output(0);
Output<Integer> fetch = g.operation("Y").output(0);
try (Tensor<Integer> x = Tensors.create(new int[][] {{5}, {7}});
TestUtil.AutoCloseableList<Tensor<?>> outputs =
new TestUtil.AutoCloseableList<Tensor<?>>(s.runner().feed(feed, x).fetch(fetch).run())) {
assertEquals(1, outputs.size());
final int[][] expected = {{31}};
assertArrayEquals(expected, outputs.get(0).copyTo(new int[1][1]));
}
}
}
@Test
public void runUsingColonSeparatedNames() {
try (Graph g = new Graph();
Session s = new Session(g)) {
Operation split =
g.opBuilder("Split", "Split")
.addInput(TestUtil.constant(g, "split_dim", 0))
.addInput(TestUtil.constant(g, "value", new int[] {1, 2, 3, 4}))
.setAttr("num_split", 2)
.build();
g.opBuilder("Add", "Add")
.addInput(split.output(0))
.addInput(split.output(1))
.build()
.output(0);
// Fetch using colon separated names.
try (Tensor<Integer> fetched =
s.runner().fetch("Split:1").run().get(0).expect(Integer.class)) {
final int[] expected = {3, 4};
assertArrayEquals(expected, fetched.copyTo(new int[2]));
}
// Feed using colon separated names.
try (Tensor<Integer> fed = Tensors.create(new int[] {4, 3, 2, 1});
Tensor<Integer> fetched =
s.runner()
.feed("Split:0", fed)
.feed("Split:1", fed)
.fetch("Add")
.run()
.get(0)
.expect(Integer.class)) {
final int[] expected = {8, 6, 4, 2};
assertArrayEquals(expected, fetched.copyTo(new int[4]));
}
}
}
@Test
public void runWithMetadata() {
try (Graph g = new Graph();
Session s = new Session(g)) {
TestUtil.transpose_A_times_X(g, new int[][] {{2}, {3}});
try (Tensor<Integer> x = Tensors.create(new int[][] {{5}, {7}})) {
Session.Run result =
s.runner()
.feed("X", x)
.fetch("Y")
.setOptions(fullTraceRunOptions())
.runAndFetchMetadata();
// Sanity check on outputs.
TestUtil.AutoCloseableList<Tensor<?>> outputs = new TestUtil.AutoCloseableList<Tensor<?>>(result.outputs);
assertEquals(1, outputs.size());
final int[][] expected = {{31}};
assertArrayEquals(expected, outputs.get(0).copyTo(new int[1][1]));
// Sanity check on metadata
// See comments in fullTraceRunOptions() for an explanation about
// why this check is really silly. Ideally, this would be:
/*
RunMetadata md = RunMetadata.parseFrom(result.metadata);
assertTrue(md.toString(), md.hasStepStats());
*/
assertTrue(result.metadata.length > 0);
outputs.close();
}
}
}
@Test
public void runMultipleOutputs() {
try (Graph g = new Graph();
Session s = new Session(g)) {
TestUtil.constant(g, "c1", 2718);
TestUtil.constant(g, "c2", 31415);
TestUtil.AutoCloseableList<Tensor<?>> outputs =
new TestUtil.AutoCloseableList<Tensor<?>>(s.runner().fetch("c2").fetch("c1").run());
assertEquals(2, outputs.size());
assertEquals(31415, outputs.get(0).intValue());
assertEquals(2718, outputs.get(1).intValue());
outputs.close();
}
}
@Test
public void failOnUseAfterClose() {
try (Graph g = new Graph()) {
Session s = new Session(g);
s.close();
try {
s.runner().run();
fail("methods on a session should fail after close() is called");
} catch (IllegalStateException e) {
// expected exception
}
}
}
@Test
public void createWithConfigProto() {
try (Graph g = new Graph();
Session s = new Session(g, singleThreadConfigProto())) {}
}
private static byte[] fullTraceRunOptions() {
// Ideally this would use the generated Java sources for protocol buffers
// and end up with something like the snippet below. However, generating
// the Java files for the .proto files in tensorflow/core:protos_all is
// a bit cumbersome in bazel until the proto_library rule is setup.
//
// See https://github.com/bazelbuild/bazel/issues/52#issuecomment-194341866
// https://github.com/bazelbuild/rules_go/pull/121#issuecomment-251515362
// https://github.com/bazelbuild/rules_go/pull/121#issuecomment-251692558
//
// For this test, for now, the use of specific bytes suffices.
return new byte[] {0x08, 0x03};
/*
return org.tensorflow.framework.RunOptions.newBuilder()
.setTraceLevel(RunOptions.TraceLevel.FULL_TRACE)
.build()
.toByteArray();
*/
}
public static byte[] singleThreadConfigProto() {
// Ideally this would use the generated Java sources for protocol buffers
// and end up with something like the snippet below. However, generating
// the Java files for the .proto files in tensorflow/core:protos_all is
// a bit cumbersome in bazel until the proto_library rule is setup.
//
// See https://github.com/bazelbuild/bazel/issues/52#issuecomment-194341866
// https://github.com/bazelbuild/rules_go/pull/121#issuecomment-251515362
// https://github.com/bazelbuild/rules_go/pull/121#issuecomment-251692558
//
// For this test, for now, the use of specific bytes suffices.
return new byte[] {0x10, 0x01, 0x28, 0x01};
/*
return org.tensorflow.framework.ConfigProto.newBuilder()
.setInterOpParallelismThreads(1)
.setIntraOpParallelismThreads(1)
.build()
.toByteArray();
*/
}
}
tensorflow/java/src/test/java/org/tensorflow/ShapeTest.java
+104
View File
@@ -0,0 +1,104 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link Shape}. */
@RunWith(JUnit4.class)
public class ShapeTest {
@Test
public void unknown() {
assertEquals(-1, Shape.unknown().numDimensions());
assertEquals("<unknown>", Shape.unknown().toString());
}
@Test
public void scalar() {
assertEquals(0, Shape.scalar().numDimensions());
assertEquals("[]", Shape.scalar().toString());
}
@Test
public void make() {
Shape s = Shape.make(2);
assertEquals(1, s.numDimensions());
assertEquals(2, s.size(0));
assertEquals("[2]", s.toString());
s = Shape.make(2, 3);
assertEquals(2, s.numDimensions());
assertEquals(2, s.size(0));
assertEquals(3, s.size(1));
assertEquals("[2, 3]", s.toString());
s = Shape.make(-1, 2, 3);
assertEquals(3, s.numDimensions());
assertEquals(-1, s.size(0));
assertEquals(2, s.size(1));
assertEquals(3, s.size(2));
assertEquals("[?, 2, 3]", s.toString());
}
@Test
public void nodesInAGraph() {
try (Graph g = new Graph()) {
Output<Float> n = TestUtil.placeholder(g, "feed", Float.class);
assertEquals(-1, n.shape().numDimensions());
n = TestUtil.constant(g, "scalar", 3);
assertEquals(0, n.shape().numDimensions());
n = TestUtil.constant(g, "vector", new float[2]);
assertEquals(1, n.shape().numDimensions());
assertEquals(2, n.shape().size(0));
n = TestUtil.constant(g, "matrix", new float[4][5]);
assertEquals(2, n.shape().numDimensions());
assertEquals(4, n.shape().size(0));
assertEquals(5, n.shape().size(1));
}
}
@Test
public void equalsWorksCorrectly() {
assertEquals(Shape.scalar(), Shape.scalar());
assertEquals(Shape.make(1, 2, 3), Shape.make(1, 2, 3));
assertNotEquals(Shape.make(1, 2), null);
assertNotEquals(Shape.make(1, 2), new Object());
assertNotEquals(Shape.make(1, 2, 3), Shape.make(1, 2, 4));
assertNotEquals(Shape.unknown(), Shape.unknown());
assertNotEquals(Shape.make(-1), Shape.make(-1));
assertNotEquals(Shape.make(1, -1, 3), Shape.make(1, -1, 3));
}
@Test
public void hashCodeIsAsExpected() {
assertEquals(Shape.make(1, 2, 3, 4).hashCode(), Shape.make(1, 2, 3, 4).hashCode());
assertEquals(Shape.scalar().hashCode(), Shape.scalar().hashCode());
assertEquals(Shape.unknown().hashCode(), Shape.unknown().hashCode());
assertNotEquals(Shape.make(1, 2).hashCode(), Shape.make(1, 3).hashCode());
}
}
tensorflow/java/src/test/java/org/tensorflow/TensorFlowTest.java
+62
View File
@@ -0,0 +1,62 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link org.tensorflow.TensorFlow}. */
@RunWith(JUnit4.class)
public class TensorFlowTest {
@Test
public void version() {
assertTrue(TensorFlow.version().length() > 0);
}
@Test
public void registeredOpList() {
// Would be nice to actually parse the output as a tensorflow.OpList protocol buffer message,
// but as of May 2017, bazel support for generating Java code from protocol buffer definitions
// was not sorted out. Revisit? Till then, at least exercise the code.
assertTrue(TensorFlow.registeredOpList().length > 0);
}
@Test
public void loadLibrary() {
// TODO(ashankar): This tell will fail when built with --config=monolithic.
// Figure out how we can ignore the test in that case.
try (Graph g = new Graph()) {
// Build a graph with an unrecognized operation.
try {
g.opBuilder("MyTest", "MyTest").build();
fail("should not be able to construct graphs with unregistered ops");
} catch (IllegalArgumentException e) {
// expected exception
}
// Load the library containing the operation.
byte[] opList = TensorFlow.loadLibrary("tensorflow/java/my_test_op.so");
assertTrue(opList.length > 0);
// Now graph building should succeed.
g.opBuilder("MyTest", "MyTest").build();
}
}
}
tensorflow/java/src/test/java/org/tensorflow/TensorTest.java
+593
View File
@@ -0,0 +1,593 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.DoubleBuffer;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.nio.LongBuffer;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.tensorflow.types.UInt8;
/** Unit tests for {@link org.tensorflow.Tensor}. */
@RunWith(JUnit4.class)
public class TensorTest {
private static final double EPSILON = 1e-7;
private static final float EPSILON_F = 1e-7f;
@Test
public void createWithByteBuffer() {
double[] doubles = {1d, 2d, 3d, 4d};
long[] doubles_shape = {4};
boolean[] bools = {true, false, true, false};
long[] bools_shape = {4};
byte[] bools_ = TestUtil.bool2byte(bools);
byte[] strings = "test".getBytes(UTF_8);
long[] strings_shape = {};
byte[] strings_; // raw TF_STRING
try (Tensor<String> t = Tensors.create(strings)) {
ByteBuffer to = ByteBuffer.allocate(t.numBytes());
t.writeTo(to);
strings_ = to.array();
}
// validate creating a tensor using a byte buffer
{
try (Tensor<Boolean> t = Tensor.create(Boolean.class, bools_shape, ByteBuffer.wrap(bools_))) {
boolean[] actual = t.copyTo(new boolean[bools_.length]);
for (int i = 0; i < bools.length; ++i) {
assertEquals("" + i, bools[i], actual[i]);
}
}
// note: the buffer is expected to contain raw TF_STRING (as per C API)
try (Tensor<String> t =
Tensor.create(String.class, strings_shape, ByteBuffer.wrap(strings_))) {
assertArrayEquals(strings, t.bytesValue());
}
}
// validate creating a tensor using a direct byte buffer (in host order)
{
ByteBuffer buf = ByteBuffer.allocateDirect(8 * doubles.length).order(ByteOrder.nativeOrder());
buf.asDoubleBuffer().put(doubles);
try (Tensor<Double> t = Tensor.create(Double.class, doubles_shape, buf)) {
double[] actual = new double[doubles.length];
assertArrayEquals(doubles, t.copyTo(actual), EPSILON);
}
}
// validate shape checking
try (Tensor<Boolean> t =
Tensor.create(Boolean.class, new long[bools_.length * 2], ByteBuffer.wrap(bools_))) {
fail("should have failed on incompatible buffer");
} catch (IllegalArgumentException e) {
// expected
}
}
@Test
public void createFromBufferWithNonNativeByteOrder() {
double[] doubles = {1d, 2d, 3d, 4d};
DoubleBuffer buf =
ByteBuffer.allocate(8 * doubles.length)
.order(
ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN
? ByteOrder.BIG_ENDIAN
: ByteOrder.LITTLE_ENDIAN)
.asDoubleBuffer()
.put(doubles);
flipBuffer(buf);
try (Tensor<Double> t = Tensor.create(new long[] {doubles.length}, buf)) {
double[] actual = new double[doubles.length];
assertArrayEquals(doubles, t.copyTo(actual), EPSILON);
}
}
@Test
public void createWithTypedBuffer() {
int[] ints = {1, 2, 3, 4};
float[] floats = {1f, 2f, 3f, 4f};
double[] doubles = {1d, 2d, 3d, 4d};
long[] longs = {1L, 2L, 3L, 4L};
long[] shape = {4};
// validate creating a tensor using a typed buffer
{
try (Tensor<Double> t = Tensor.create(shape, DoubleBuffer.wrap(doubles))) {
double[] actual = new double[doubles.length];
assertArrayEquals(doubles, t.copyTo(actual), EPSILON);
}
try (Tensor<Float> t = Tensor.create(shape, FloatBuffer.wrap(floats))) {
float[] actual = new float[floats.length];
assertArrayEquals(floats, t.copyTo(actual), EPSILON_F);
}
try (Tensor<Integer> t = Tensor.create(shape, IntBuffer.wrap(ints))) {
int[] actual = new int[ints.length];
assertArrayEquals(ints, t.copyTo(actual));
}
try (Tensor<Long> t = Tensor.create(shape, LongBuffer.wrap(longs))) {
long[] actual = new long[longs.length];
assertArrayEquals(longs, t.copyTo(actual));
}
}
// validate shape-checking
{
try (Tensor<Double> t =
Tensor.create(new long[doubles.length + 1], DoubleBuffer.wrap(doubles))) {
fail("should have failed on incompatible buffer");
} catch (IllegalArgumentException e) {
// expected
}
try (Tensor<Float> t = Tensor.create(new long[floats.length + 1], FloatBuffer.wrap(floats))) {
fail("should have failed on incompatible buffer");
} catch (IllegalArgumentException e) {
// expected
}
try (Tensor<Integer> t = Tensor.create(new long[ints.length + 1], IntBuffer.wrap(ints))) {
fail("should have failed on incompatible buffer");
} catch (IllegalArgumentException e) {
// expected
}
try (Tensor<Long> t = Tensor.create(new long[longs.length + 1], LongBuffer.wrap(longs))) {
fail("should have failed on incompatible buffer");
} catch (IllegalArgumentException e) {
// expected
}
}
}
@Test
public void writeTo() {
int[] ints = {1, 2, 3};
float[] floats = {1f, 2f, 3f};
double[] doubles = {1d, 2d, 3d};
long[] longs = {1L, 2L, 3L};
boolean[] bools = {true, false, true};
try (Tensor<Integer> tints = Tensors.create(ints);
Tensor<Float> tfloats = Tensors.create(floats);
Tensor<Double> tdoubles = Tensors.create(doubles);
Tensor<Long> tlongs = Tensors.create(longs);
Tensor<Boolean> tbools = Tensors.create(bools)) {
// validate that any datatype is readable with ByteBuffer (content, position)
{
ByteBuffer bbuf = ByteBuffer.allocate(1024).order(ByteOrder.nativeOrder());
clearBuffer(bbuf); // FLOAT
tfloats.writeTo(bbuf);
assertEquals(tfloats.numBytes(), bbuf.position());
flipBuffer(bbuf);
assertEquals(floats[0], bbuf.asFloatBuffer().get(0), EPSILON);
clearBuffer(bbuf); // DOUBLE
tdoubles.writeTo(bbuf);
assertEquals(tdoubles.numBytes(), bbuf.position());
flipBuffer(bbuf);
assertEquals(doubles[0], bbuf.asDoubleBuffer().get(0), EPSILON);
clearBuffer(bbuf); // INT32
tints.writeTo(bbuf);
assertEquals(tints.numBytes(), bbuf.position());
flipBuffer(bbuf);
assertEquals(ints[0], bbuf.asIntBuffer().get(0));
clearBuffer(bbuf); // INT64
tlongs.writeTo(bbuf);
assertEquals(tlongs.numBytes(), bbuf.position());
flipBuffer(bbuf);
assertEquals(longs[0], bbuf.asLongBuffer().get(0));
clearBuffer(bbuf); // BOOL
tbools.writeTo(bbuf);
assertEquals(tbools.numBytes(), bbuf.position());
flipBuffer(bbuf);
assertEquals(bools[0], bbuf.get(0) != 0);
}
// validate the use of direct buffers
{
DoubleBuffer buf =
ByteBuffer.allocateDirect(tdoubles.numBytes())
.order(ByteOrder.nativeOrder())
.asDoubleBuffer();
tdoubles.writeTo(buf);
assertTrue(buf.isDirect());
assertEquals(tdoubles.numElements(), buf.position());
assertEquals(doubles[0], buf.get(0), EPSILON);
}
// validate typed buffers (content, position)
{
FloatBuffer buf = FloatBuffer.allocate(tfloats.numElements());
tfloats.writeTo(buf);
assertEquals(tfloats.numElements(), buf.position());
assertEquals(floats[0], buf.get(0), EPSILON);
}
{
DoubleBuffer buf = DoubleBuffer.allocate(tdoubles.numElements());
tdoubles.writeTo(buf);
assertEquals(tdoubles.numElements(), buf.position());
assertEquals(doubles[0], buf.get(0), EPSILON);
}
{
IntBuffer buf = IntBuffer.allocate(tints.numElements());
tints.writeTo(buf);
assertEquals(tints.numElements(), buf.position());
assertEquals(ints[0], buf.get(0));
}
{
LongBuffer buf = LongBuffer.allocate(tlongs.numElements());
tlongs.writeTo(buf);
assertEquals(tlongs.numElements(), buf.position());
assertEquals(longs[0], buf.get(0));
}
// validate byte order conversion
{
DoubleBuffer foreignBuf =
ByteBuffer.allocate(tdoubles.numBytes())
.order(
ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN
? ByteOrder.BIG_ENDIAN
: ByteOrder.LITTLE_ENDIAN)
.asDoubleBuffer();
tdoubles.writeTo(foreignBuf);
flipBuffer(foreignBuf);
double[] actual = new double[foreignBuf.remaining()];
foreignBuf.get(actual);
assertArrayEquals(doubles, actual, EPSILON);
}
// validate that incompatible buffers are rejected
{
IntBuffer badbuf1 = IntBuffer.allocate(128);
try {
tbools.writeTo(badbuf1);
fail("should have failed on incompatible buffer");
} catch (IllegalArgumentException e) {
// expected
}
FloatBuffer badbuf2 = FloatBuffer.allocate(128);
try {
tbools.writeTo(badbuf2);
fail("should have failed on incompatible buffer");
} catch (IllegalArgumentException e) {
// expected
}
DoubleBuffer badbuf3 = DoubleBuffer.allocate(128);
try {
tbools.writeTo(badbuf3);
fail("should have failed on incompatible buffer");
} catch (IllegalArgumentException e) {
// expected
}
LongBuffer badbuf4 = LongBuffer.allocate(128);
try {
tbools.writeTo(badbuf4);
fail("should have failed on incompatible buffer");
} catch (IllegalArgumentException e) {
// expected
}
}
}
}
@Test
public void scalars() {
try (Tensor<Float> t = Tensors.create(2.718f)) {
assertEquals(DataType.FLOAT, t.dataType());
assertEquals(0, t.numDimensions());
assertEquals(0, t.shape().length);
assertEquals(2.718f, t.floatValue(), EPSILON_F);
}
try (Tensor<Double> t = Tensors.create(3.1415)) {
assertEquals(DataType.DOUBLE, t.dataType());
assertEquals(0, t.numDimensions());
assertEquals(0, t.shape().length);
assertEquals(3.1415, t.doubleValue(), EPSILON);
}
try (Tensor<Integer> t = Tensors.create(-33)) {
assertEquals(DataType.INT32, t.dataType());
assertEquals(0, t.numDimensions());
assertEquals(0, t.shape().length);
assertEquals(-33, t.intValue());
}
try (Tensor<Long> t = Tensors.create(8589934592L)) {
assertEquals(DataType.INT64, t.dataType());
assertEquals(0, t.numDimensions());
assertEquals(0, t.shape().length);
assertEquals(8589934592L, t.longValue());
}
try (Tensor<Boolean> t = Tensors.create(true)) {
assertEquals(DataType.BOOL, t.dataType());
assertEquals(0, t.numDimensions());
assertEquals(0, t.shape().length);
assertTrue(t.booleanValue());
}
final byte[] bytes = {1, 2, 3, 4};
try (Tensor<String> t = Tensors.create(bytes)) {
assertEquals(DataType.STRING, t.dataType());
assertEquals(0, t.numDimensions());
assertEquals(0, t.shape().length);
assertArrayEquals(bytes, t.bytesValue());
}
}
@Test
public void nDimensional() {
double[] vector = {1.414, 2.718, 3.1415};
try (Tensor<Double> t = Tensors.create(vector)) {
assertEquals(DataType.DOUBLE, t.dataType());
assertEquals(1, t.numDimensions());
assertArrayEquals(new long[] {3}, t.shape());
double[] got = new double[3];
assertArrayEquals(vector, t.copyTo(got), EPSILON);
}
int[][] matrix = {{1, 2, 3}, {4, 5, 6}};
try (Tensor<Integer> t = Tensors.create(matrix)) {
assertEquals(DataType.INT32, t.dataType());
assertEquals(2, t.numDimensions());
assertArrayEquals(new long[] {2, 3}, t.shape());
int[][] got = new int[2][3];
assertArrayEquals(matrix, t.copyTo(got));
}
long[][][] threeD = {
{{1}, {3}, {5}, {7}, {9}}, {{2}, {4}, {6}, {8}, {0}},
};
try (Tensor<Long> t = Tensors.create(threeD)) {
assertEquals(DataType.INT64, t.dataType());
assertEquals(3, t.numDimensions());
assertArrayEquals(new long[] {2, 5, 1}, t.shape());
long[][][] got = new long[2][5][1];
assertArrayEquals(threeD, t.copyTo(got));
}
boolean[][][][] fourD = {
{{{false, false, false, true}, {false, false, true, false}}},
{{{false, false, true, true}, {false, true, false, false}}},
{{{false, true, false, true}, {false, true, true, false}}},
};
try (Tensor<Boolean> t = Tensors.create(fourD)) {
assertEquals(DataType.BOOL, t.dataType());
assertEquals(4, t.numDimensions());
assertArrayEquals(new long[] {3, 1, 2, 4}, t.shape());
boolean[][][][] got = new boolean[3][1][2][4];
assertArrayEquals(fourD, t.copyTo(got));
}
}
@Test
public void testNDimensionalStringTensor() {
byte[][][] matrix = new byte[4][3][];
for (int i = 0; i < 4; ++i) {
for (int j = 0; j < 3; ++j) {
matrix[i][j] = String.format("(%d, %d) = %d", i, j, i << j).getBytes(UTF_8);
}
}
try (Tensor<String> t = Tensors.create(matrix)) {
assertEquals(DataType.STRING, t.dataType());
assertEquals(2, t.numDimensions());
assertArrayEquals(new long[] {4, 3}, t.shape());
byte[][][] got = t.copyTo(new byte[4][3][]);
assertEquals(4, got.length);
for (int i = 0; i < 4; ++i) {
assertEquals(String.format("%d", i), 3, got[i].length);
for (int j = 0; j < 3; ++j) {
assertArrayEquals(String.format("(%d, %d)", i, j), matrix[i][j], got[i][j]);
}
}
}
}
@Test
public void testUInt8Tensor() {
byte[] vector = new byte[] {1, 2, 3, 4};
try (Tensor<UInt8> t = Tensor.create(vector, UInt8.class)) {
assertEquals(DataType.UINT8, t.dataType());
assertEquals(1, t.numDimensions());
assertArrayEquals(new long[] {4}, t.shape());
byte[] got = t.copyTo(new byte[4]);
assertArrayEquals(vector, got);
}
}
@Test
public void testCreateFromArrayOfBoxed() {
Integer[] vector = new Integer[] {1, 2, 3, 4};
try (Tensor<Integer> t = Tensor.create(vector, Integer.class)) {
fail("Tensor.create() should fail because it was given an array of boxed values");
} catch (IllegalArgumentException e) {
// The expected exception
}
}
@Test
public void failCreateOnMismatchedDimensions() {
int[][][] invalid = new int[3][1][];
for (int x = 0; x < invalid.length; ++x) {
for (int y = 0; y < invalid[x].length; ++y) {
invalid[x][y] = new int[x + y + 1];
}
}
try (Tensor<?> t = Tensor.create(invalid)) {
fail("Tensor.create() should fail because of differing sizes in the 3rd dimension");
} catch (IllegalArgumentException e) {
// The expected exception.
}
}
@Test
public void failCopyToOnIncompatibleDestination() {
try (final Tensor<Integer> matrix = Tensors.create(new int[][] {{1, 2}, {3, 4}})) {
try {
matrix.copyTo(new int[2]);
fail("should have failed on dimension mismatch");
} catch (IllegalArgumentException e) {
// The expected exception.
}
try {
matrix.copyTo(new float[2][2]);
fail("should have failed on DataType mismatch");
} catch (IllegalArgumentException e) {
// The expected exception.
}
try {
matrix.copyTo(new int[2][3]);
fail("should have failed on shape mismatch");
} catch (IllegalArgumentException e) {
// The expected exception.
}
}
}
@Test
public void failCopyToOnScalar() {
try (final Tensor<Integer> scalar = Tensors.create(3)) {
try {
scalar.copyTo(3);
fail("copyTo should fail on scalar tensors, suggesting use of primitive accessors instead");
} catch (IllegalArgumentException e) {
// The expected exception.
}
}
}
@Test
public void failOnArbitraryObject() {
try (Tensor<?> t = Tensor.create(new Object())) {
fail("should fail on creating a Tensor with a Java object that has no equivalent DataType");
} catch (IllegalArgumentException e) {
// The expected exception.
}
}
@Test
public void failOnZeroDimension() {
try (Tensor<Integer> t = Tensors.create(new int[3][0][1])) {
fail("should fail on creating a Tensor where one of the dimensions is 0");
} catch (IllegalArgumentException e) {
// The expected exception.
}
}
@Test
public void useAfterClose() {
int n = 4;
Tensor<?> t = Tensor.create(n);
t.close();
try {
t.intValue();
} catch (NullPointerException e) {
// The expected exception.
}
}
@Test
public void eagerTensorIsReleasedAfterSessionIsClosed() {
Tensor<Integer> sum;
try (EagerSession session = EagerSession.create()) {
Output<?> x = TestUtil.constant(session, "Const1", 10);
Output<?> y = TestUtil.constant(session, "Const2", 20);
sum = TestUtil.<Integer>addN(session, x, y).tensor();
assertNotEquals(0L, sum.getNativeHandle());
assertEquals(30, sum.intValue());
}
assertEquals(0L, sum.getNativeHandle());
try {
sum.intValue();
fail();
} catch (NullPointerException e) {
// expected.
}
}
@Test
public void fromHandle() {
// fromHandle is a package-visible method intended for use when the C TF_Tensor object has been
// created independently of the Java code. In practice, two Tensor instances MUST NOT have the
// same native handle.
//
// An exception is made for this test, where the pitfalls of this is avoided by not calling
// close() on both Tensors.
final float[][] matrix = {{1, 2, 3}, {4, 5, 6}};
try (Tensor<Float> src = Tensors.create(matrix)) {
Tensor<Float> cpy = Tensor.fromHandle(src.getNativeHandle()).expect(Float.class);
assertEquals(src.dataType(), cpy.dataType());
assertEquals(src.numDimensions(), cpy.numDimensions());
assertArrayEquals(src.shape(), cpy.shape());
assertArrayEquals(matrix, cpy.copyTo(new float[2][3]));
}
}
@Test
public void gracefullyFailCreationFromNullArrayForStringTensor() {
// Motivated by: https://github.com/tensorflow/tensorflow/issues/17130
byte[][] array = new byte[1][];
try {
Tensors.create(array);
} catch (NullPointerException e) {
// expected.
}
}
// Workaround for cross compiliation
// (e.g., javac -source 1.9 -target 1.8).
//
// In Java 8 and prior, subclasses of java.nio.Buffer (e.g., java.nio.DoubleBuffer) inherited the
// "flip()" and "clear()" methods from java.nio.Buffer resulting in the signature:
// Buffer flip();
// In Java 9 these subclasses had their own methods like:
// DoubleBuffer flip();
// As a result, compiling for 1.9 source for a target of JDK 1.8 would result in errors at runtime
// like:
//
// java.lang.NoSuchMethodError: java.nio.DoubleBuffer.flip()Ljava/nio/DoubleBuffer
private static void flipBuffer(Buffer buf) {
buf.flip();
}
// See comment for flipBuffer()
private static void clearBuffer(Buffer buf) {
buf.clear();
}
}
tensorflow/java/src/test/java/org/tensorflow/TestUtil.java
+178
View File
@@ -0,0 +1,178 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Collection;
/** Static utility functions. */
public class TestUtil {
public static final class AutoCloseableList<E extends AutoCloseable> extends ArrayList<E>
implements AutoCloseable {
public AutoCloseableList(Collection<? extends E> c) {
super(c);
}
@Override
public void close() {
Exception toThrow = null;
for (AutoCloseable c : this) {
try {
c.close();
} catch (Exception e) {
toThrow = e;
}
}
if (toThrow != null) {
throw new RuntimeException(toThrow);
}
}
}
public static GraphOperation constantOp(Graph g, String name, Object value) {
try (Tensor<?> t = Tensor.create(value)) {
return g.opBuilder("Const", name).setAttr("dtype", t.dataType()).setAttr("value", t).build();
}
}
public static <T> Output<T> constant(ExecutionEnvironment env, String name, Object value) {
try (Tensor<?> t = Tensor.create(value)) {
return env.opBuilder("Const", name)
.setAttr("dtype", t.dataType())
.setAttr("value", t)
.build()
.<T>output(0);
}
}
public static <T> Output<T> placeholder(Graph g, String name, Class<T> type) {
return g.opBuilder("Placeholder", name)
.setAttr("dtype", DataType.fromClass(type))
.build()
.<T>output(0);
}
public static <T> Output<T> addN(ExecutionEnvironment env, Output<?>... inputs) {
return env.opBuilder("AddN", "AddN").addInputList(inputs).build().output(0);
}
public static <T> Output<T> matmul(
Graph g, String name, Output<T> a, Output<T> b, boolean transposeA, boolean transposeB) {
return g.opBuilder("MatMul", name)
.addInput(a)
.addInput(b)
.setAttr("transpose_a", transposeA)
.setAttr("transpose_b", transposeB)
.build()
.<T>output(0);
}
public static Operation split(Graph g, String name, int[] values, int numSplit) {
return g.opBuilder("Split", name)
.addInput(constant(g, "split_dim", 0))
.addInput(constant(g, "values", values))
.setAttr("num_split", numSplit)
.build();
}
public static <T> Output<T> square(Graph g, String name, Output<T> value) {
return g.opBuilder("Square", name)
.addInput(value)
.build()
.<T>output(0);
}
public static void transpose_A_times_X(Graph g, int[][] a) {
Output<Integer> aa = constant(g, "A", a);
matmul(g, "Y", aa, placeholder(g, "X", Integer.class), true, false);
}
/**
* Counts the total number of elements in an ND array.
*
* @param array the array to count the elements of
* @return the number of elements
*/
public static int flattenedNumElements(Object array) {
int count = 0;
for (int i = 0; i < Array.getLength(array); i++) {
Object e = Array.get(array, i);
if (!e.getClass().isArray()) {
count += 1;
} else {
count += flattenedNumElements(e);
}
}
return count;
}
/**
* Flattens an ND-array into a 1D-array with the same elements.
*
* @param array the array to flatten
* @param elementType the element class (e.g. {@code Integer.TYPE} for an {@code int[]})
* @return a flattened array
*/
public static Object flatten(Object array, Class<?> elementType) {
Object out = Array.newInstance(elementType, flattenedNumElements(array));
flatten(array, out, 0);
return out;
}
private static int flatten(Object array, Object out, int next) {
for (int i = 0; i < Array.getLength(array); i++) {
Object e = Array.get(array, i);
if (!e.getClass().isArray()) {
Array.set(out, next++, e);
} else {
next = flatten(e, out, next);
}
}
return next;
}
/**
* Converts a {@code boolean[]} to a {@code byte[]}.
*
* <p>Suitable for creating tensors of type {@link DataType#BOOL} using {@link
* java.nio.ByteBuffer}.
*/
public static byte[] bool2byte(boolean[] array) {
byte[] out = new byte[array.length];
for (int i = 0; i < array.length; i++) {
out[i] = array[i] ? (byte) 1 : (byte) 0;
}
return out;
}
/**
* Converts a {@code byte[]} to a {@code boolean[]}.
*
* <p>Suitable for reading tensors of type {@link DataType#BOOL} using {@link
* java.nio.ByteBuffer}.
*/
public static boolean[] byte2bool(byte[] array) {
boolean[] out = new boolean[array.length];
for (int i = 0; i < array.length; i++) {
out[i] = array[i] != 0;
}
return out;
}
private TestUtil() {}
}

Some files were not shown because too many files have changed in this diff Show More