wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions 3 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/python/kernel_tests/nn_ops/BUILD
+925
View File
@@ -0,0 +1,925 @@
# Tests of TensorFlow NN kernels written using the Python API.
load("@xla//third_party/rules_python/python:py_library.bzl", "py_library")
load("//tensorflow:tensorflow.default.bzl", "cuda_py_strict_test", "tf_py_strict_test")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//tensorflow:internal"],
licenses = ["notice"],
)
CONV_TEST_BASENAMES = [
":atrous_conv2d_test",
":conv1d_test",
":conv1d_transpose_test",
":conv2d_backprop_filter_grad_test",
":conv3d_transpose_test",
":conv3d_backprop_filter_v2_grad_test",
":conv_ops_3d_test",
":conv_ops_test",
":depthwise_conv_op_test",
":conv2d_transpose_test",
]
test_suite(
name = "conv_tests",
tests = [basename for basename in CONV_TEST_BASENAMES] +
[basename + "_gpu" for basename in CONV_TEST_BASENAMES],
)
cuda_py_strict_test(
name = "atrous_conv2d_test",
size = "medium",
srcs = ["atrous_conv2d_test.py"],
shard_count = 2,
tags = [
"no_gpu", # Flaky: b/80127739, b/127001953
],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_impl",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "atrous_convolution_test",
size = "medium",
srcs = ["atrous_convolution_test.py"],
tags = [
"manual",
"no_cuda_asan",
],
deps = [
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "betainc_op_test",
size = "small",
srcs = ["betainc_op_test.py"],
tags = [
"notap", # TODO(delhibabu): Re-enable once the test is fixed.
],
xla_tags = [
"no_cuda_asan", # times out
],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:gradients_impl",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:tf_logging",
"//third_party/py/numpy",
],
)
py_library(
name = "bias_op_base",
srcs = ["bias_op_base.py"],
strict_deps = True,
deps = [
"//tensorflow/python/eager:backprop",
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:gradient_checker_v2",
"//tensorflow/python/ops:gradients_impl",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "bias_op_d9m_test",
size = "medium",
srcs = ["bias_op_d9m_test.py"],
shard_count = 2,
deps = [
":bias_op_base",
"//tensorflow/python/eager:backprop",
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:config",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:gradients_impl",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"@absl_py//absl/testing:parameterized",
],
)
cuda_py_strict_test(
name = "bias_op_test",
size = "medium",
srcs = ["bias_op_test.py"],
deps = [
":bias_op_base",
"//tensorflow/python/platform:client_testlib",
],
)
cuda_py_strict_test(
name = "conv1d_test",
size = "small",
srcs = ["conv1d_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:array_ops_stack",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
],
)
cuda_py_strict_test(
name = "conv1d_transpose_test",
size = "small",
srcs = ["conv1d_transpose_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "conv2d_backprop_filter_grad_test",
size = "medium",
srcs = ["conv2d_backprop_filter_grad_test.py"],
shard_count = 2,
tags = [
"optonly", # flaky timeouts unless optimized
],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "conv2d_transpose_test",
size = "small",
srcs = ["conv2d_transpose_test.py"],
# TODO(b/144432983): S32 convolutions should not be auto-clustered, only
# crashes tests.
xla_enable_strict_auto_jit = False,
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:array_ops_stack",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/ops:random_ops",
"//tensorflow/python/ops:variable_scope",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "conv3d_backprop_filter_v2_grad_test",
size = "small",
srcs = ["conv3d_backprop_filter_v2_grad_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "conv3d_transpose_test",
size = "medium",
srcs = ["conv3d_transpose_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "conv_ops_3d_test",
size = "medium",
srcs = ["conv_ops_3d_test.py"],
shard_count = 30,
tags = [
"no_cuda11",
"no_mac_arm64",
],
deps = [
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:tensor_shape",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:gradients_impl",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/ops:nn_ops_gen",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:tf_logging",
"//tensorflow/python/util:compat",
"@absl_py//absl/testing:parameterized",
],
)
cuda_py_strict_test(
name = "conv_ops_test",
size = "medium",
srcs = ["conv_ops_test.py"],
shard_count = 4,
tags = [
"no_mac_arm64",
"optonly", # times out
],
deps = [
"//tensorflow/core:protos_all_py",
"//tensorflow/python/client:session",
"//tensorflow/python/eager:backprop",
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:control_flow_ops",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:gradients_impl",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_impl",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/ops:nn_ops_gen",
"//tensorflow/python/ops:random_ops",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:tf_logging",
"//tensorflow/python/util:compat",
"//third_party/py/numpy",
"@absl_py//absl/testing:parameterized",
],
)
tf_py_strict_test(
name = "ctc_decoder_ops_test",
size = "small",
srcs = ["ctc_decoder_ops_test.py"],
deps = [
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops_stack",
"//tensorflow/python/ops:ctc_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "ctc_loss_op_test",
size = "medium",
srcs = ["ctc_loss_op_test.py"],
xla_enable_strict_auto_jit = False, # b/148153828
deps = [
"//tensorflow/python/eager:backprop",
"//tensorflow/python/eager:context",
"//tensorflow/python/eager:def_function",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:random_seed",
"//tensorflow/python/framework:sparse_tensor",
"//tensorflow/python/framework:tensor_spec",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:ctc_ops",
"//tensorflow/python/ops:gradients_impl",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:random_ops",
"//tensorflow/python/ops:sparse_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
"@absl_py//absl/testing:parameterized",
],
)
py_library(
name = "cudnn_deterministic_base",
srcs = ["cudnn_deterministic_base.py"],
strict_deps = True,
deps = [
"//tensorflow/python/eager:backprop",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "cudnn_deterministic_ops_test",
size = "small",
srcs = ["cudnn_deterministic_ops_test.py"],
tags = [
"no_cuda_asan", # TODO(b/171509035): re-enable.
"no_rocm_pre_53",
],
xla_enable_strict_auto_jit = True,
deps = [
":cudnn_deterministic_base",
"//tensorflow/python/framework:config",
"//tensorflow/python/platform:client_testlib",
],
)
cuda_py_strict_test(
name = "cudnn_d9m_test",
size = "small",
srcs = ["cudnn_d9m_test.py"],
tags = [
"no_cuda_asan", # TODO(b/171509035): re-enable.
"no_rocm", #This is test is specific to CUDA and enables determinism through a CUDA specific env var.
],
deps = [
":cudnn_deterministic_base",
"//tensorflow/python/platform:client_testlib",
],
)
py_library(
name = "depthwise_conv_op_base",
srcs = ["depthwise_conv_op_base.py"],
strict_deps = True,
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_impl",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:tf_logging",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "depthwise_conv_op_d9m_test",
size = "medium", # http://b/30603882
timeout = "long",
srcs = ["depthwise_conv_op_d9m_test.py"],
shard_count = 8,
deps = [
":depthwise_conv_op_base",
"//tensorflow/python/eager:backprop",
"//tensorflow/python/framework:config",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:random_seed",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_impl",
"//tensorflow/python/ops:random_ops",
"//tensorflow/python/platform:client_testlib",
],
)
cuda_py_strict_test(
name = "depthwise_conv_op_test",
size = "medium", # http://b/30603882
timeout = "long",
srcs = ["depthwise_conv_op_test.py"],
shard_count = 8,
deps = [
":depthwise_conv_op_base",
"//tensorflow/python/platform:client_testlib",
],
)
cuda_py_strict_test(
name = "embedding_ops_test",
size = "medium",
srcs = ["embedding_ops_test.py"],
shard_count = 20,
tags = [
"no_cuda_asan", # Size limit: b/192505612
],
xla_tags = [
"no_cuda_asan", # Size limit: b/192505612
],
deps = [
"//tensorflow/python/compat",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:sparse_tensor",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:array_ops_stack",
"//tensorflow/python/ops:data_flow_ops",
"//tensorflow/python/ops:embedding_ops",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:gradients",
"//tensorflow/python/ops:init_ops",
"//tensorflow/python/ops:linalg_ops",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:partitioned_variables",
"//tensorflow/python/ops:resource_variable_ops",
"//tensorflow/python/ops:sort_ops",
"//tensorflow/python/ops:state_ops",
"//tensorflow/python/ops:variable_scope",
"//tensorflow/python/ops:variables",
"//tensorflow/python/ops/ragged:ragged_factory_ops",
"//tensorflow/python/ops/ragged:ragged_tensor",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:tf_logging",
"//tensorflow/python/util:compat",
"//third_party/py/numpy",
"@absl_py//absl/testing:parameterized",
],
)
tf_py_strict_test(
name = "fractional_avg_pool_op_test",
size = "small",
srcs = ["fractional_avg_pool_op_test.py"],
shard_count = 5,
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/ops:nn_ops_gen",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
tf_py_strict_test(
name = "fractional_max_pool_op_test",
size = "small",
srcs = ["fractional_max_pool_op_test.py"],
shard_count = 5,
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/ops:nn_ops_gen",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
tf_py_strict_test(
name = "losses_test",
size = "medium",
srcs = ["losses_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:random_seed",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:init_ops",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:random_ops",
"//tensorflow/python/ops:variable_scope",
"//tensorflow/python/ops:variables",
"//tensorflow/python/ops/losses",
"//tensorflow/python/ops/losses:util",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/training:momentum",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "lrn_op_test",
size = "medium",
srcs = ["lrn_op_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:gradients_impl",
"//tensorflow/python/ops:nn",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:random_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "morphological_ops_test",
size = "small",
srcs = ["morphological_ops_test.py"],
xla_tags = [
"no_cuda_asan", # times out
],
deps = [
"//tensorflow/python/framework:config",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:gradient_checker_v2",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
"@absl_py//absl/testing:parameterized",
],
)
cuda_py_strict_test(
name = "nth_element_op_test",
size = "small",
srcs = ["nth_element_op_test.py"],
deps = [
"//tensorflow/python/eager:backprop",
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:gradients_impl",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "pool_test",
size = "medium",
srcs = ["pool_test.py"],
xla_tags = [
"no_cuda_asan", # times out
],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "pooling_ops_3d_test",
size = "medium",
srcs = ["pooling_ops_3d_test.py"],
deps = [
"//tensorflow/python/eager:backprop",
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:gradient_checker_v2",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/ops:nn_ops_gen",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "pooling_ops_test",
size = "medium",
srcs = ["pooling_ops_test.py"],
shard_count = 10,
# Some operations in this test can only be checked on sm61+.
tags = ["prefer-sm70"],
deps = [
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:config",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:array_ops_gen",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:gradients_impl",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/ops:nn_ops_gen",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:tf_logging",
"//third_party/py/numpy",
"@absl_py//absl/testing:parameterized",
],
)
cuda_py_strict_test(
name = "relu_op_test",
size = "small",
srcs = ["relu_op_test.py"],
xla_tags = [
"no_cuda_asan", # times out
],
deps = [
"//tensorflow/python/eager:backprop",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:gradient_checker_v2",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/ops:random_ops",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/training:gradient_descent",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "rnn_cell_test",
size = "medium",
srcs = ["rnn_cell_test.py"],
shard_count = 15,
tags = ["no_windows"], # b/139739217
deps = [
"//tensorflow/core:protos_all_py",
"//tensorflow/python/checkpoint",
"//tensorflow/python/eager:context",
"//tensorflow/python/eager:def_function",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:random_seed",
"//tensorflow/python/framework:tensor",
"//tensorflow/python/framework:tensor_shape",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:array_ops_stack",
"//tensorflow/python/ops:cond",
"//tensorflow/python/ops:gradients_impl",
"//tensorflow/python/ops:init_ops",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:random_ops",
"//tensorflow/python/ops:rnn",
"//tensorflow/python/ops:rnn_cell",
"//tensorflow/python/ops:rnn_cell_impl",
"//tensorflow/python/ops:rnn_ops_gen",
"//tensorflow/python/ops:state_ops",
"//tensorflow/python/ops:tensor_array_ops",
"//tensorflow/python/ops:variable_scope",
"//tensorflow/python/ops:variable_v1",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:tf_logging",
"//tensorflow/python/saved_model:load",
"//tensorflow/python/saved_model:save",
"//tensorflow/python/trackable:autotrackable",
"//tensorflow/python/util:nest",
"//third_party/py/numpy",
"@absl_py//absl/testing:parameterized",
],
)
cuda_py_strict_test(
name = "rnn_test",
size = "medium",
timeout = "long",
srcs = ["rnn_test.py"],
shard_count = 10,
deps = [
"//tensorflow/core:protos_all_py",
"//tensorflow/python/client:session",
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:array_ops_stack",
"//tensorflow/python/ops:control_flow_ops",
"//tensorflow/python/ops:data_flow_grad",
"//tensorflow/python/ops:gradients_impl",
"//tensorflow/python/ops:init_ops",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:rnn",
"//tensorflow/python/ops:rnn_cell_impl",
"//tensorflow/python/ops:sparse_grad",
"//tensorflow/python/ops:tensor_array_grad",
"//tensorflow/python/ops:tensor_array_ops",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/training:saver",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "softmax_op_test",
size = "medium",
srcs = ["softmax_op_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:tf_logging",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "softplus_op_test",
size = "small",
srcs = ["softplus_op_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:gradients_impl",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "softsign_op_test",
size = "small",
srcs = ["softsign_op_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "xent_op_d9m_test",
size = "medium",
timeout = "long",
srcs = ["xent_op_d9m_test.py"],
tags = [
"no_pip", # TODO(b/283889547) Flaky on kokoro
"no_windows", # Flaky on Windows CPU: https://github.com/tensorflow/tensorflow/issues/55827
"nomac", # TODO(b/235277289) Flaky on OSX
],
xla_enable_strict_auto_jit = False,
deps = [
":xent_op_test_base",
"//tensorflow/python/eager:backprop",
"//tensorflow/python/framework:config",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/ops:nn_ops_gen",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "xent_op_test",
size = "small",
srcs = ["xent_op_test.py"],
deps = [
":xent_op_test_base",
"//tensorflow/python/client:session",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/ops:nn_ops_gen",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
py_library(
name = "xent_op_test_base",
srcs = ["xent_op_test_base.py"],
strict_deps = True,
deps = [
"//tensorflow/python/eager:backprop",
"//tensorflow/python/framework:config",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:gradient_checker",
"//tensorflow/python/ops:gradients_impl",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:nn_grad",
"//tensorflow/python/ops:nn_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
tensorflow/python/kernel_tests/nn_ops/atrous_conv2d_test.py
+252
View File
@@ -0,0 +1,252 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for convolution related functionality in tensorflow.ops.nn."""
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import nn_impl
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
def _upsample_filters(filters, rate):
"""Upsamples the filters by a factor of rate along the spatial dimensions.
Args:
filters: [h, w, in_depth, out_depth]. Original filters.
rate: An int, specifying the upsampling rate.
Returns:
filters_up: [h_up, w_up, in_depth, out_depth]. Upsampled filters with
h_up = h + (h - 1) * (rate - 1)
w_up = w + (w - 1) * (rate - 1)
containing (rate - 1) zeros between consecutive filter values along
the filters' spatial dimensions.
"""
if rate == 1:
return filters
# [h, w, in_depth, out_depth] -> [in_depth, out_depth, h, w]
filters_up = np.transpose(filters, [2, 3, 0, 1])
ker = np.zeros([rate, rate], dtype=np.float32)
ker[0, 0] = 1
filters_up = np.kron(filters_up, ker)[:, :, :-(rate - 1), :-(rate - 1)]
# [in_depth, out_depth, h_up, w_up] -> [h_up, w_up, in_depth, out_depth]
filters_up = np.transpose(filters_up, [2, 3, 0, 1])
return filters_up
class AtrousConv2DTest(test.TestCase):
@test_util.run_deprecated_v1
def testAtrousConv2DForward(self):
with self.session():
# Input: [batch, height, width, input_depth]
height = 9
for width in [9, 10]: # Test both odd and even width.
x_shape = [2, height, width, 2]
x = np.arange(np.prod(x_shape), dtype=np.float32).reshape(x_shape)
# Filter: [kernel_height, kernel_width, input_depth, output_depth]
for kernel_height in range(1, 4):
for kernel_width in range(1, 4):
f_shape = [kernel_height, kernel_width, 2, 2]
f = np.arange(np.prod(f_shape), dtype=np.float32).reshape(f_shape)
for rate in range(1, 4):
f_up = _upsample_filters(f, rate)
for padding in ["SAME", "VALID"]:
y1 = nn_ops.atrous_conv2d(x, f, rate, padding=padding)
y2 = nn_ops.conv2d(
x, f_up, strides=[1, 1, 1, 1], padding=padding)
self.assertAllClose(y1, y2, rtol=1e-3, atol=1e-3)
@test_util.run_deprecated_v1
def testAtrousSequence(self):
"""Tests optimization of sequence of atrous convolutions.
Verifies that a sequence of `atrous_conv2d` operations with identical `rate`
parameters, 'SAME' `padding`, and `filters` with odd heights/ widths:
net = atrous_conv2d(net, filters1, rate, padding="SAME")
net = atrous_conv2d(net, filters2, rate, padding="SAME")
...
net = atrous_conv2d(net, filtersK, rate, padding="SAME")
is equivalent to:
pad = ... # padding so that the input dims are multiples of rate
net = space_to_batch(net, paddings=pad, block_size=rate)
net = conv2d(net, filters1, strides=[1, 1, 1, 1], padding="SAME")
net = conv2d(net, filters2, strides=[1, 1, 1, 1], padding="SAME")
...
net = conv2d(net, filtersK, strides=[1, 1, 1, 1], padding="SAME")
net = batch_to_space(net, crops=pad, block_size=rate)
"""
padding = "SAME" # The padding needs to be "SAME"
np.random.seed(1) # Make it reproducible.
with self.session():
# Input: [batch, height, width, input_depth]
for height in range(15, 17):
for width in range(15, 17):
x_shape = [3, height, width, 2]
x = np.random.random_sample(x_shape).astype(np.float32)
for kernel in [1, 3, 5]: # The kernel size needs to be odd.
# Filter: [kernel_height, kernel_width, input_depth, output_depth]
f_shape = [kernel, kernel, 2, 2]
f = 1e-2 * np.random.random_sample(f_shape).astype(np.float32)
for rate in range(2, 4):
# y1: three atrous_conv2d in a row.
y1 = nn_ops.atrous_conv2d(x, f, rate, padding=padding)
y1 = nn_ops.atrous_conv2d(y1, f, rate, padding=padding)
y1 = nn_ops.atrous_conv2d(y1, f, rate, padding=padding)
# y2: space_to_batch, three conv2d in a row, batch_to_space
pad_bottom = 0 if height % rate == 0 else rate - height % rate
pad_right = 0 if width % rate == 0 else rate - width % rate
pad = [[0, pad_bottom], [0, pad_right]]
y2 = array_ops.space_to_batch(x, paddings=pad, block_size=rate)
y2 = nn_ops.conv2d(y2, f, strides=[1, 1, 1, 1], padding=padding)
y2 = nn_ops.conv2d(y2, f, strides=[1, 1, 1, 1], padding=padding)
y2 = nn_ops.conv2d(y2, f, strides=[1, 1, 1, 1], padding=padding)
y2 = array_ops.batch_to_space(y2, crops=pad, block_size=rate)
self.assertAllClose(y1, y2, rtol=1e-2, atol=1e-2)
@test_util.run_deprecated_v1
def testGradient(self):
with self.session():
# Input: [batch, height, width, input_depth]
x_shape = [2, 5, 6, 2]
# Filter: [kernel_height, kernel_width, input_depth, output_depth]
f_shape = [3, 3, 2, 2]
# Output: [batch, height, width, output_depth]
y_shape = [2, 5, 6, 2]
np.random.seed(1) # Make it reproducible.
x_val = np.random.random_sample(x_shape).astype(np.float32)
f_val = np.random.random_sample(f_shape).astype(np.float32)
x = constant_op.constant(x_val, name="x", dtype=dtypes.float32)
f = constant_op.constant(f_val, name="f", dtype=dtypes.float32)
for rate in range(1, 4):
output = nn_ops.atrous_conv2d(x, f, rate=rate, padding="SAME")
err = gradient_checker.compute_gradient_error([x, f],
[x_shape, f_shape],
output, y_shape)
print("atrous_conv2d gradient err = %g " % err)
err_tolerance = 4e-3 if test_util.is_xla_enabled() else 1e-3
self.assertLess(err, err_tolerance)
@test_util.run_deprecated_v1
def testAtrousConv2DInvalid(self):
with self.session():
with self.assertRaises((errors.InvalidArgumentError, ValueError)):
op = nn_ops.atrous_conv2d(
value=np.ones((1, 1, 1, 5)),
filters=np.ones((1, 1, 5, 1)),
rate=2147483647,
padding="SAME")
self.evaluate(op)
class AtrousConv2DTransposeTest(test.TestCase):
@test_util.run_deprecated_v1
def testAtrousConv2DTransposeForward(self):
with self.session():
# Input: [batch, height, width, input_depth]
height = 9
for width in [9, 10]: # Test both odd and even width.
x_shape = [2, height, width, 2]
x = np.arange(np.prod(x_shape), dtype=np.float32).reshape(x_shape)
# Filter: [kernel_height, kernel_width, input_depth, output_depth]
for kernel_height in range(1, 4):
for kernel_width in range(1, 4):
f_shape = [kernel_height, kernel_width, 2, 2]
f = np.arange(np.prod(f_shape), dtype=np.float32).reshape(f_shape)
for rate in range(1, 4):
f_up = _upsample_filters(f, rate)
kernel_height_up = (kernel_height + (kernel_height - 1) *
(rate - 1))
kernel_width_up = kernel_width + (kernel_width - 1) * (rate - 1)
for padding in ["SAME", "VALID"]:
if padding == "SAME":
y_shape = [2, height, width, 2]
else:
y_shape = [
2, height + kernel_height_up - 1,
width + kernel_width_up - 1, 2
]
y1 = nn_ops.atrous_conv2d_transpose(x, f, y_shape, rate,
padding)
y2 = nn_ops.conv2d_transpose(
x, f_up, y_shape, strides=[1, 1, 1, 1], padding=padding)
self.assertAllClose(y1, y2, rtol=1e-3, atol=1e-3)
def testAtrousConv2DTransposeInvalid(self):
with self.session():
with self.assertRaises((errors.InvalidArgumentError, ValueError)):
op = nn_ops.atrous_conv2d_transpose(
value=np.ones((10, 1, 1, 1)),
filters=np.ones((1, 1, 1, 1)),
rate=1356819205,
padding="SAME",
output_shape=[1, 1, 1, 1])
self.evaluate(op)
class AtrousDepthwiseConv2DTest(test.TestCase):
@test_util.run_deprecated_v1
def testAtrousDepthwiseConv2DForward(self):
strides = [1, 1, 1, 1]
with self.session():
# Input: [batch, height, width, input_depth]
height = 9
for width in [9, 10]: # Test both odd and even width.
x_shape = [2, height, width, 2]
x = np.arange(np.prod(x_shape), dtype=np.float32).reshape(x_shape)
# Filter: [kernel_height, kernel_width, input_depth, output_depth]
for kernel_height in range(1, 4):
for kernel_width in range(1, 4):
f_shape = [kernel_height, kernel_width, 2, 2]
f = np.arange(np.prod(f_shape), dtype=np.float32).reshape(f_shape)
for rate in range(1, 4):
f_up = _upsample_filters(f, rate)
for padding in ["SAME", "VALID"]:
y1 = nn_impl.depthwise_conv2d(
x, f, strides, padding, rate=[rate, rate])
y2 = nn_impl.depthwise_conv2d(x, f_up, strides, padding)
self.assertAllClose(y1, y2, rtol=1e-3, atol=1e-3)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/atrous_convolution_test.py
+277
View File
@@ -0,0 +1,277 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for atrous convolution functionality in tensorflow.ops.nn."""
import contextlib
import numpy as np
from tensorflow.python.eager import context
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
def upsample_filters(filters, rate):
"""Upsamples the filters by a factor of rate along the spatial dimensions.
Args:
filters: spatial_shape + [in_channels, out_channels]
Original filters.
rate: A list of len(spatial_shape) positive ints, specifying the
upsampling rate.
Returns:
filters_up: output_spatial_shape + [in_channels, out_channels].
Upsampled filters with
output_spatial_shape[i] = (spatial_shape[i] - 1) * rate[i] + 1
containing (rate[i] - 1) zeros between consecutive filter values along
spatial dimension i.
"""
num_spatial_dims = len(rate)
spatial_shape = np.array(filters.shape[:num_spatial_dims])
output_spatial_shape = (spatial_shape - 1) * rate + 1
output = np.zeros(
tuple(output_spatial_shape) + tuple(filters.shape[-2:]), filters.dtype)
output[tuple(np.s_[::rate[i]] for i in range(num_spatial_dims))] = filters
return output
class AtrousConvolutionTest(test.TestCase):
@contextlib.contextmanager
def _delay_checks(self):
"""Context manager for combining checks depending on tensor evaluations.
Each call to Session.run has some overhead, and this overhead can easily
account for the majority of the time spent in tests that call Session.run
(or Tensor.eval) many times.
This context manager provides a mechanism for registering callback functions
and associated tensors. When the context is exited, all of the tensors
associated with all of the registrations are evaluated with a single call to
Session.run, and then each registered callback function is called with the
values of its associated tensors.
Yields:
A function `add_check(check, *args, **kwargs)` where `check` is the
callback function to be invoked, and `*args` and `**kwargs` specify the
associated Tensors. When in EAGER mode, check is executed in add_check,
otherwise, it's delayed after the context.
"""
checks = []
def add_check(check, *args, **kwargs):
if context.executing_eagerly():
args_val, kwargs_val = self.evaluate([args, kwargs])
check(*args_val, **kwargs_val)
else:
checks.append((check, args, kwargs))
yield add_check
if not context.executing_eagerly():
all_values = self.evaluate([[args, kwargs] for _, args, kwargs in checks])
for (check, _, _), (args, kwargs) in zip(checks, all_values):
check(*args, **kwargs)
def _test_atrous_convolution(self, add_check, input_shape, filter_shape,
dilation_rate, **kwargs):
filters = np.arange(
np.prod(filter_shape), dtype=np.float32).reshape(filter_shape)
filters_upsampled = upsample_filters(filters, dilation_rate)
x = np.arange(np.prod(input_shape), dtype=np.float32).reshape(input_shape)
y1 = nn_ops.convolution(
input=x, filter=filters, dilation_rate=dilation_rate, **kwargs)
y2 = nn_ops.convolution(input=x, filter=filters_upsampled, **kwargs)
def check(y1_eval, y2_eval):
self.assertAllClose(y1_eval, y2_eval, rtol=1e-2, atol=1e-2)
add_check(check, y1, y2)
@test_util.run_v1_only("b/120545219")
def test_unknown_spatial_dims_for_channel_last_format(self):
x = array_ops.placeholder(dtypes.float32, [1, None, None, 10])
w = array_ops.zeros([3, 3, 10, 20])
y = nn_ops.convolution(
x, w, "VALID", dilation_rate=[2, 2], data_format="NHWC")
self.assertEqual(y.shape.as_list(), [1, None, None, 20])
@test_util.run_v1_only("b/120545219")
def test_unknown_spatial_dims_for_channel_first_format(self):
x = array_ops.placeholder(dtypes.float32, [1, 10, None, None])
w = array_ops.zeros([3, 3, 10, 20])
y = nn_ops.convolution(
x, w, "VALID", dilation_rate=[2, 2], data_format="NCHW")
self.assertEqual(y.shape.as_list(), [1, 20, None, None])
@test_util.run_in_graph_and_eager_modes
def testAtrousConvolution2D(self):
with self._delay_checks() as add_check:
for padding in ["SAME", "VALID"]:
for height, width in [[9, 9], [9, 10]]:
for kernel_height, kernel_width in [[1, 1], [2, 2], [2, 3]]:
for dilation_rate in [[1, 1], [3, 2], [2, 1]]:
self._test_atrous_convolution(
add_check=add_check,
input_shape=[2, height, width, 2],
filter_shape=[kernel_height, kernel_width, 2, 2],
padding=padding,
dilation_rate=dilation_rate,
)
@test_util.run_in_graph_and_eager_modes
def testAtrousConvolution3D(self):
with self._delay_checks() as add_check:
for padding in ["SAME", "VALID"]:
for depth, height, width in [[9, 9, 10], [9, 10, 9]]:
for kernel_depth, kernel_height, kernel_width in [[3, 3,
3], [3, 2, 2],
[2, 1, 3]]:
for dilation_rate in [[1, 1, 1], [3, 3, 3], [3, 2, 3], [3, 1, 2]]:
self._test_atrous_convolution(
add_check=add_check,
input_shape=[2, depth, height, width, 2],
filter_shape=[
kernel_depth, kernel_height, kernel_width, 2, 2
],
padding=padding,
dilation_rate=dilation_rate,
)
@test_util.run_in_graph_and_eager_modes
def testAtrousConvolution1D(self):
with self._delay_checks() as add_check:
for padding in ["SAME", "VALID"]:
for width in [9, 10]:
for kernel_width in range(1, 4):
for rate in range(1, 4):
self._test_atrous_convolution(
add_check=add_check,
input_shape=[2, width, 2],
filter_shape=[kernel_width, 2, 2],
padding=padding,
dilation_rate=[rate],
)
@test_util.run_in_graph_and_eager_modes
def testAtrousConvolutionNC(self):
if test.is_gpu_available(cuda_only=True):
# "NCW" and "NCHW" formats are currently supported only on CUDA.
with test_util.device(use_gpu=True):
with self._delay_checks() as add_check:
for padding in ["SAME", "VALID"]:
self._test_atrous_convolution(
add_check=add_check,
input_shape=[2, 2, 9],
padding=padding,
filter_shape=[3, 2, 2],
dilation_rate=[2],
data_format="NCW",
)
self._test_atrous_convolution(
add_check=add_check,
input_shape=[2, 2, 9, 5],
padding=padding,
filter_shape=[3, 3, 2, 2],
dilation_rate=[2, 1],
data_format="NCHW",
)
@test_util.run_in_graph_and_eager_modes
def testAtrousSequence(self):
"""Tests optimization of sequence of atrous convolutions.
See the documentation of with_space_to_batch.
"""
with self._delay_checks() as add_check:
for padding in ["SAME", "VALID"]:
for height in range(15, 17):
for width in range(15, 17):
x_shape = [3, height, width, 2]
x = np.random.random_sample(x_shape).astype(np.float32)
kernel_sizes = [1, 3] if padding == "SAME" else range(1, 3)
for kernel in kernel_sizes:
f_shape = [kernel, kernel, 2, 2]
f1 = 1e-2 * np.random.random_sample(f_shape).astype(np.float32)
f2 = 1e-2 * np.random.random_sample(f_shape).astype(np.float32)
def combined_op(converted_input, num_spatial_dims, padding_arg): # pylint: disable=unused-argument
# pylint: disable=cell-var-from-loop
result = nn_ops.convolution(
input=converted_input, filter=f1, padding=padding)
result = nn_ops.convolution(
input=result, filter=f2, padding=padding)
# pylint: enable=cell-var-from-loop
return result
for rate_height in range(2, 4):
for rate_width in range(2, 4):
dilation_rate = [rate_height, rate_width]
y1 = nn_ops.convolution(
input=x,
filter=f1,
padding=padding,
dilation_rate=dilation_rate)
y1 = nn_ops.convolution(
input=y1,
filter=f2,
padding=padding,
dilation_rate=dilation_rate)
y2 = nn_ops.with_space_to_batch(
input=x,
dilation_rate=dilation_rate,
op=combined_op,
padding="VALID")
def check(y1_eval, y2_eval):
self.assertAllClose(y1_eval, y2_eval, rtol=1e-2, atol=1e-2)
add_check(check, y1, y2)
def _test_gradient(self, x_shape, f_shape, dilation_rate, padding):
x_val = np.random.random_sample(x_shape).astype(np.float32)
f_val = np.random.random_sample(f_shape).astype(np.float32)
x = constant_op.constant(x_val, name="x", dtype=dtypes.float32)
f = constant_op.constant(f_val, name="f", dtype=dtypes.float32)
output = nn_ops.convolution(
input=x, filter=f, dilation_rate=dilation_rate, padding=padding)
y_shape = output.get_shape().as_list()
err = gradient_checker.compute_gradient_error([x, f], [x_shape, f_shape],
output, y_shape)
err_tolerance = 1e-2
self.assertLess(err, err_tolerance)
@test_util.run_v1_only("b/120545219")
def testGradient(self):
with self.cached_session():
for padding in ["SAME", "VALID"]:
for rate_width in range(1, 3):
for rate_height in range(1, 3):
self._test_gradient(
x_shape=[2, 5, 6, 2],
f_shape=[3, 3, 2, 2],
dilation_rate=[rate_height, rate_width],
padding=padding)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/betainc_op_test.py
+207
View File
@@ -0,0 +1,207 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Functional tests for 3d convolutional operations."""
import itertools
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import gradients_impl
from tensorflow.python.ops import math_ops
from tensorflow.python.platform import test
from tensorflow.python.platform import tf_logging
class BetaincTest(test.TestCase):
def _testBetaInc(self, a_s, b_s, x_s, dtype):
try:
from scipy import special # pylint: disable=g-import-not-at-top
np_dt = dtype.as_numpy_dtype
# Test random values
a_s = a_s.astype(np_dt) # in (0, infty)
b_s = b_s.astype(np_dt) # in (0, infty)
x_s = x_s.astype(np_dt) # in (0, 1)
tf_a_s = constant_op.constant(a_s, dtype=dtype)
tf_b_s = constant_op.constant(b_s, dtype=dtype)
tf_x_s = constant_op.constant(x_s, dtype=dtype)
tf_out_t = math_ops.betainc(tf_a_s, tf_b_s, tf_x_s)
with self.cached_session():
tf_out = self.evaluate(tf_out_t)
scipy_out = special.betainc(a_s, b_s, x_s, dtype=np_dt)
# the scipy version of betainc uses a double-only implementation.
# TODO(ebrevdo): identify reasons for (sometime) precision loss
# with doubles
rtol = 1e-4
atol = 1e-5
self.assertAllCloseAccordingToType(
scipy_out, tf_out, rtol=rtol, atol=atol)
# Test out-of-range values. Keep these expectations explicit rather than
# relying on SciPy behavior for invalid-domain inputs.
combinations = list(itertools.product([-1, 0, 0.5, 1.0, 1.5], repeat=3))
a_comb, b_comb, x_comb = np.asarray(list(zip(*combinations)), dtype=np_dt)
with self.cached_session():
tf_comb = math_ops.betainc(a_comb, b_comb, x_comb).eval()
invalid_mask = (
(a_comb < 0)
| (b_comb < 0)
| ((a_comb == 0) & (b_comb == 0))
| (x_comb < 0)
| (x_comb > 1)
)
self.assertAllEqual(
np.ones_like(tf_comb[invalid_mask], dtype=np.bool_),
np.isnan(tf_comb[invalid_mask]),
)
# Test broadcasting between scalars and other shapes
with self.cached_session():
self.assertAllCloseAccordingToType(
special.betainc(0.1, b_s, x_s, dtype=np_dt),
math_ops.betainc(0.1, b_s, x_s).eval(),
rtol=rtol,
atol=atol)
self.assertAllCloseAccordingToType(
special.betainc(a_s, 0.1, x_s, dtype=np_dt),
math_ops.betainc(a_s, 0.1, x_s).eval(),
rtol=rtol,
atol=atol)
self.assertAllCloseAccordingToType(
special.betainc(a_s, b_s, 0.1, dtype=np_dt),
math_ops.betainc(a_s, b_s, 0.1).eval(),
rtol=rtol,
atol=atol)
self.assertAllCloseAccordingToType(
special.betainc(0.1, b_s, 0.1, dtype=np_dt),
math_ops.betainc(0.1, b_s, 0.1).eval(),
rtol=rtol,
atol=atol)
self.assertAllCloseAccordingToType(
special.betainc(0.1, 0.1, 0.1, dtype=np_dt),
math_ops.betainc(0.1, 0.1, 0.1).eval(),
rtol=rtol,
atol=atol)
with self.assertRaisesRegex(ValueError, "must be equal"):
math_ops.betainc(0.5, [0.5], [[0.5]])
with self.cached_session():
with self.assertRaisesOpError("Shapes of .* are inconsistent"):
a_p = array_ops.placeholder(dtype)
b_p = array_ops.placeholder(dtype)
x_p = array_ops.placeholder(dtype)
math_ops.betainc(a_p, b_p, x_p).eval(
feed_dict={a_p: 0.5,
b_p: [0.5],
x_p: [[0.5]]})
except ImportError as e:
tf_logging.warn("Cannot test special functions: %s" % str(e))
@test_util.run_deprecated_v1
def testBetaIncFloat(self):
a_s = np.abs(np.random.randn(10, 10) * 30) # in (0, infty)
b_s = np.abs(np.random.randn(10, 10) * 30) # in (0, infty)
x_s = np.random.rand(10, 10) # in (0, 1)
self._testBetaInc(a_s, b_s, x_s, dtypes.float32)
@test_util.run_deprecated_v1
def testBetaIncDouble(self):
a_s = np.abs(np.random.randn(10, 10) * 30) # in (0, infty)
b_s = np.abs(np.random.randn(10, 10) * 30) # in (0, infty)
x_s = np.random.rand(10, 10) # in (0, 1)
self._testBetaInc(a_s, b_s, x_s, dtypes.float64)
@test_util.run_deprecated_v1
def testBetaIncDoubleVeryLargeValues(self):
a_s = np.abs(np.random.randn(10, 10) * 1e15) # in (0, infty)
b_s = np.abs(np.random.randn(10, 10) * 1e15) # in (0, infty)
x_s = np.random.rand(10, 10) # in (0, 1)
self._testBetaInc(a_s, b_s, x_s, dtypes.float64)
@test_util.run_deprecated_v1
@test_util.disable_xla("b/178338235")
def testBetaIncDoubleVerySmallValues(self):
a_s = np.abs(np.random.randn(10, 10) * 1e-16) # in (0, infty)
b_s = np.abs(np.random.randn(10, 10) * 1e-16) # in (0, infty)
x_s = np.random.rand(10, 10) # in (0, 1)
self._testBetaInc(a_s, b_s, x_s, dtypes.float64)
@test_util.run_deprecated_v1
@test_util.disable_xla("b/178338235")
def testBetaIncFloatVerySmallValues(self):
a_s = np.abs(np.random.randn(10, 10) * 1e-8) # in (0, infty)
b_s = np.abs(np.random.randn(10, 10) * 1e-8) # in (0, infty)
x_s = np.random.rand(10, 10) # in (0, 1)
self._testBetaInc(a_s, b_s, x_s, dtypes.float32)
@test_util.run_deprecated_v1
def testBetaIncFpropAndBpropAreNeverNAN(self):
with self.cached_session() as sess:
space = np.logspace(-8, 5).tolist()
space_x = np.linspace(1e-16, 1 - 1e-16).tolist()
ga_s, gb_s, gx_s = zip(*list(itertools.product(space, space, space_x)))
# Test grads are never nan
ga_s_t = constant_op.constant(ga_s, dtype=dtypes.float32)
gb_s_t = constant_op.constant(gb_s, dtype=dtypes.float32)
gx_s_t = constant_op.constant(gx_s, dtype=dtypes.float32)
tf_gout_t = math_ops.betainc(ga_s_t, gb_s_t, gx_s_t)
tf_gout, grads_x = sess.run(
[tf_gout_t,
gradients_impl.gradients(tf_gout_t, [ga_s_t, gb_s_t, gx_s_t])[2]])
# Equivalent to `assertAllFalse` (if it existed).
self.assertAllEqual(
np.zeros_like(grads_x).astype(np.bool_), np.isnan(tf_gout))
self.assertAllEqual(
np.zeros_like(grads_x).astype(np.bool_), np.isnan(grads_x))
@test_util.run_deprecated_v1
def testBetaIncGrads(self):
err_tolerance = 1e-3
with self.cached_session():
# Test gradient
ga_s = np.abs(np.random.randn(2, 2) * 30) # in (0, infty)
gb_s = np.abs(np.random.randn(2, 2) * 30) # in (0, infty)
gx_s = np.random.rand(2, 2) # in (0, 1)
tf_ga_s = constant_op.constant(ga_s, dtype=dtypes.float64)
tf_gb_s = constant_op.constant(gb_s, dtype=dtypes.float64)
tf_gx_s = constant_op.constant(gx_s, dtype=dtypes.float64)
tf_gout_t = math_ops.betainc(tf_ga_s, tf_gb_s, tf_gx_s)
err = gradient_checker.compute_gradient_error(
[tf_gx_s], [gx_s.shape], tf_gout_t, gx_s.shape)
tf_logging.info("betainc gradient err = %g " % err)
self.assertLess(err, err_tolerance)
# Test broadcast gradient
gx_s = np.random.rand() # in (0, 1)
tf_gx_s = constant_op.constant(gx_s, dtype=dtypes.float64)
tf_gout_t = math_ops.betainc(tf_ga_s, tf_gb_s, tf_gx_s)
err = gradient_checker.compute_gradient_error(
[tf_gx_s], [()], tf_gout_t, ga_s.shape)
tf_logging.info("betainc gradient err = %g " % err)
self.assertLess(err, err_tolerance)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/bias_op_base.py
+322
View File
@@ -0,0 +1,322 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Functional tests for BiasAdd."""
import numpy as np
from tensorflow.python.eager import backprop
from tensorflow.python.eager import context
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import gradient_checker_v2
from tensorflow.python.ops import gradients_impl
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
@test_util.run_all_in_graph_and_eager_modes
class BiasAddTestBase(test.TestCase):
def _npBias(self, inputs, bias):
assert len(bias.shape) == 1
assert inputs.shape[-1] == bias.shape[0]
return inputs + bias.reshape(([1] *
(len(inputs.shape) - 1)) + [bias.shape[0]])
def testNpBias(self):
self.assertAllClose(
np.array([[11, 22, 33], [41, 52, 63]]),
self._npBias(
np.array([[10, 20, 30], [40, 50, 60]]), np.array([1, 2, 3])))
def _testBias(self, np_inputs, np_bias, use_gpu=False):
np_val = self._npBias(np_inputs, np_bias)
with self.cached_session(use_gpu=use_gpu):
tf_val = self.evaluate(nn_ops.bias_add(np_inputs, np_bias))
self.assertAllCloseAccordingToType(np_val, tf_val)
def _AtLeast3d(self, np_value):
# fill the input value to at least 3-dimension
if np_value.ndim < 3:
return np.reshape(np_value, (1,) * (3 - np_value.ndim) + np_value.shape)
return np_value
def _NHWCToNCHW(self, np_value):
# fill the input value to at least 3-dimension
np_value = self._AtLeast3d(np_value)
# move the last dimension to second
np_dim = list(range(np_value.ndim))
np_dim_new = list(np_dim[0:1]) + list(np_dim[-1:]) + list(np_dim[1:-1])
return np.transpose(np_value, np_dim_new)
def _NCHWToNHWC(self, np_value):
assert len(np_value.shape) >= 3
np_dim = list(range(np_value.ndim))
# move the second dimension to the last
np_dim_new = list(np_dim[0:1]) + list(np_dim[2:]) + list(np_dim[1:2])
return np.transpose(np_value, np_dim_new)
def _testBiasNCHW(self, np_inputs, np_bias, use_gpu):
np_val = self._npBias(np_inputs, np_bias)
np_inputs = self._NHWCToNCHW(np_inputs)
with self.cached_session(use_gpu=use_gpu):
tf_val = self.evaluate(
nn_ops.bias_add(np_inputs, np_bias, data_format="NCHW"))
tf_val = self._NCHWToNHWC(tf_val)
self.assertAllCloseAccordingToType(self._AtLeast3d(np_val), tf_val)
def _testAll(self, np_inputs, np_bias):
self._testBias(np_inputs, np_bias, use_gpu=False)
self._testBiasNCHW(np_inputs, np_bias, use_gpu=False)
if np_inputs.dtype in [np.float16, np.float32, np.float64, np.int32]:
self._testBias(np_inputs, np_bias, use_gpu=True)
self._testBiasNCHW(np_inputs, np_bias, use_gpu=True)
def _expectedException(self):
if context.executing_eagerly():
return errors_impl.InvalidArgumentError
else:
return ValueError
def testInputDims(self):
with self.assertRaises(self._expectedException()):
nn_ops.bias_add([1, 2], [1])
def testBiasVec(self):
with self.assertRaises(self._expectedException()):
nn_ops.bias_add(
array_ops.reshape([1, 2], shape=[1, 2]),
array_ops.reshape([1, 2], shape=[1, 2]))
def testBiasInputsMatch(self):
with self.assertRaises(self._expectedException()):
nn_ops.bias_add(
array_ops.reshape([1, 2], shape=[1, 2]),
array_ops.reshape([1], shape=[1]))
def testIntTypes(self):
for t in [np.int8, np.int16, np.int32, np.int64]:
self._testAll(
np.array([[10, 20, 30], [40, 50, 60]]).astype(t),
np.array([1, 2, 3]).astype(t))
def testFloatTypes(self):
for t in [
np.float16, np.float32, np.float64, dtypes.bfloat16.as_numpy_dtype
]:
self._testAll(
np.random.rand(4, 3, 3).astype(t),
np.random.rand(3).astype(t))
def test4DFloatTypes(self):
for t in [
np.float16, np.float32, np.float64, dtypes.bfloat16.as_numpy_dtype
]:
self._testAll(
np.random.rand(4, 3, 2, 3).astype(t),
np.random.rand(3).astype(t))
self._testAll(
np.random.rand(2048, 4, 4, 4).astype(t),
np.random.rand(4).astype(t))
self._testAll(
np.random.rand(4, 4, 4, 2048).astype(t),
np.random.rand(2048).astype(t))
def test5DFloatTypes(self):
for t in [
np.float16, np.float32, np.float64, dtypes.bfloat16.as_numpy_dtype
]:
self._testAll(
np.random.rand(4, 3, 2, 3, 4).astype(t),
np.random.rand(4).astype(t))
def _random_tensor(self, shape, dtype):
return constant_op.constant(2 * np.random.rand(*shape) - 1, dtype=dtype)
def _computeGradient(self, np_input, bias, dtype, data_format):
input_shape = output_shape = np_input.shape
bias_shape = bias.shape
input_tensor = constant_op.constant(
np_input, shape=input_shape, dtype=dtype)
bias_tensor = constant_op.constant(bias, shape=bias_shape, dtype=dtype)
if context.executing_eagerly():
def bias_add(input_tensor, bias_tensor):
return nn_ops.bias_add(
input_tensor, bias_tensor, data_format=data_format)
# The following is a work-around for TF issue 33660. Instead of
# calculating the analytical and numerical gradients for both
# inputs in a single call to compute_gradient, compute_gradient
# is called for each input separately.
def bias_add_1(input_tensor):
return bias_add(input_tensor, bias_tensor)
def bias_add_2(bias_tensor):
return bias_add(input_tensor, bias_tensor)
input_jacob_a, input_jacob_n = gradient_checker_v2.compute_gradient(
bias_add_1, [input_tensor])
bias_jacob_a, bias_jacob_n = gradient_checker_v2.compute_gradient(
bias_add_2, [bias_tensor])
# Test gradient of BiasAddGrad
def bias_add_grad_function(upstream_gradients):
with backprop.GradientTape() as tape:
tape.watch(bias_tensor)
bias_add_output = bias_add(input_tensor, bias_tensor)
gradient_injector_output = bias_add_output * upstream_gradients
return tape.gradient(gradient_injector_output, bias_tensor)
upstream_tensor = self._random_tensor(output_shape, dtype)
grad_jacob_a, grad_jacob_n = gradient_checker_v2.compute_gradient(
bias_add_grad_function, [upstream_tensor])
else:
output_tensor = nn_ops.bias_add(
input_tensor, bias_tensor, data_format=data_format)
jacobians = gradient_checker.compute_gradient([input_tensor, bias_tensor],
[input_shape, bias_shape],
output_tensor, output_shape)
(input_jacob_a, input_jacob_n), (bias_jacob_a, bias_jacob_n) = jacobians
# Test gradient of BiasAddGrad
if dtype == dtypes.bfloat16:
# L2Loss is not supported for bfloat16 on CPU.
output_tensor = math_ops.cast(output_tensor, dtype=dtypes.float32)
bias_add_grad = gradients_impl.gradients(
nn_ops.l2_loss(output_tensor), bias_tensor)[0]
grad_jacob_a, grad_jacob_n = gradient_checker.compute_gradient(
output_tensor, output_shape, bias_add_grad, bias_shape)
return ((input_jacob_a, bias_jacob_a, grad_jacob_a),
(input_jacob_n, bias_jacob_n, grad_jacob_n))
def _testGradient(self, np_input, bias, dtype, data_format, use_gpu):
with self.cached_session(use_gpu=use_gpu):
if data_format == "NCHW":
np_input = self._NHWCToNCHW(np_input)
jacob_a, jacob_n = self._computeGradient(np_input, bias, dtype,
data_format)
input_jacob_a, bias_jacob_a, grad_jacob_a = jacob_a
input_jacob_n, bias_jacob_n, grad_jacob_n = jacob_n
if dtype in [np.float16, dtypes.bfloat16.as_numpy_dtype]:
# Compare fp16/bf16 analytical gradients to fp32 numerical gradients,
# since fp16/bf16 numerical gradients are too imprecise unless great
# care is taken with choosing the inputs and the delta. This is
# a weaker, but pragmatic, check (in particular, it does not test
# the op itself, only its gradient).
_, jacob_n = self._computeGradient(np_input, bias, np.float32,
data_format)
input_jacob_n, bias_jacob_n, grad_jacob_n = jacob_n
if dtype == dtypes.float64:
threshold = 1e-10
elif np_input.size >= 512:
# The 5e-3 threshold seems to have been marginal in these cases, and
# small changes in the test were pushing it over the limit.
threshold = 5e-2
else:
threshold = 5e-3
self.assertAllClose(input_jacob_a, input_jacob_n, threshold, threshold)
self.assertAllClose(bias_jacob_a, bias_jacob_n, threshold, threshold)
self.assertAllClose(grad_jacob_a, grad_jacob_n, threshold, threshold)
def testGradientTensor2D(self):
for (data_format, use_gpu) in ("NHWC", False), ("NHWC", True):
for dtype in (dtypes.float16, dtypes.float32, dtypes.float64,
dtypes.bfloat16):
np_input = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0],
dtype=dtype.as_numpy_dtype).reshape(3, 2)
bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
self._testGradient(np_input, bias, dtype, data_format, use_gpu)
def testGradientTensor3D(self):
for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
("NCHW", False), ("NCHW", True)]:
for dtype in (dtypes.float16, dtypes.float32, dtypes.float64,
dtypes.bfloat16):
# pylint: disable=too-many-function-args
np_input = np.array(
[1.0, 2.0, 3.0, 4.0, 5.0, 6.0],
dtype=dtype.as_numpy_dtype).reshape(1, 3, 2)
bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
self._testGradient(np_input, bias, dtype, data_format, use_gpu)
def testGradientTensor4D(self):
for (data_format, use_gpu) in [("NHWC", False)]:
for dtype in (dtypes.float16, dtypes.float32, dtypes.float64,
dtypes.bfloat16):
np_input = np.arange(
1.0, 49.0,
dtype=dtype.as_numpy_dtype).reshape([2, 3, 4, 2]).astype(np.float32)
bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
self._testGradient(np_input, bias, dtype, data_format, use_gpu)
np_input = np.arange(
1.0, 513.0,
dtype=dtype.as_numpy_dtype).reshape([64, 2, 2,
2]).astype(np.float32)
self._testGradient(np_input, bias, dtype, data_format, use_gpu)
np_input = np.arange(
1.0, 513.0,
dtype=dtype.as_numpy_dtype).reshape([2, 2, 2,
64]).astype(np.float32)
self._testGradient(np_input,
np.random.rand(64).astype(dtype.as_numpy_dtype),
dtype, data_format, use_gpu)
def testGradientTensor5D(self):
for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
("NCHW", False), ("NCHW", True)]:
for dtype in (dtypes.float16, dtypes.float32, dtypes.float64,
dtypes.bfloat16):
np_input = np.arange(
1.0, 49.0,
dtype=dtype.as_numpy_dtype).reshape([1, 2, 3, 4,
2]).astype(np.float32)
bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
self._testGradient(np_input, bias, dtype, data_format, use_gpu)
def test1x1Image(self):
for (data_format, use_gpu) in [("NHWC", False), ("NCHW", False)]:
np_input = np.arange(1.0, 129.0).reshape([4, 1, 1, 32]).astype(np.float32)
self._testGradient(np_input,
np.random.rand(32).astype(np.float32), dtypes.float32,
data_format, use_gpu)
def testEmpty(self):
np.random.seed(7)
for shape in (0, 0), (2, 0), (0, 2), (4, 3, 0), (4, 0, 3), (0, 4, 3):
self._testAll(np.random.randn(*shape), np.random.randn(shape[-1]))
def testEmptyGradient(self):
for (data_format, use_gpu) in ("NHWC", False), ("NHWC", True):
for shape in (0, 0), (2, 0), (0, 2):
self._testGradient(
np.random.randn(*shape), np.random.randn(shape[-1]), dtypes.float64,
data_format, use_gpu)
for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
("NCHW", False), ("NCHW", True)]:
for shape in (4, 3, 0), (4, 0, 3), (0, 4, 3):
self._testGradient(
np.random.randn(*shape), np.random.randn(shape[-1]), dtypes.float64,
data_format, use_gpu)
tensorflow/python/kernel_tests/nn_ops/bias_op_d9m_test.py
+148
View File
@@ -0,0 +1,148 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Functional tests for deterministic BiasAdd."""
import numpy as np
from absl.testing import parameterized
from tensorflow.python.eager import backprop
from tensorflow.python.eager import context
from tensorflow.python.framework import config
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.kernel_tests.nn_ops import bias_op_base
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gradients_impl
from tensorflow.python.ops import nn_ops
from tensorflow.python.platform import test
class BiasAddDeterministicTest(bias_op_base.BiasAddTestBase,
parameterized.TestCase):
def _makeShapeTuple(self, batch_size, channel_count, data_rank, data_dim,
data_layout):
data_dims = data_rank * (data_dim,)
if data_layout == 'channels_first':
shape = (batch_size,) + (channel_count,) + data_dims
elif data_layout == 'channels_last':
shape = (batch_size,) + data_dims + (channel_count,)
else:
raise ValueError('Unknown data format')
return shape
def _dataFormatFromDataLayout(self, data_layout=None):
if data_layout == 'channels_first':
return 'NCHW'
elif data_layout == 'channels_last':
return 'NHWC'
else:
raise ValueError('Unknown data_layout')
def _randomNDArray(self, shape):
return 2 * np.random.random_sample(shape) - 1
def _randomDataOp(self, shape, data_type):
return constant_op.constant(self._randomNDArray(shape), dtype=data_type)
@parameterized.named_parameters(
*test_util.generate_combinations_with_testcase_name(
# With the selected layer configuration, at least in TensorFlow
# version 2.0, when data_layout='channels_last', bias_add operates
# deterministically by default. I don't know if this is true for
# all layer configurations. These cases are still being tested here,
# for completeness.
data_layout=['channels_first', 'channels_last'],
data_rank=[1, 2, 3],
data_type=[dtypes.float16, dtypes.float32, dtypes.float64]))
@test_util.run_in_graph_and_eager_modes
@test_util.run_cuda_only
def testDeterministicGradients(self, data_layout, data_rank, data_type):
with self.session(force_gpu=True):
# Using a cached_session with force_gpu=True does not work at the time
# of writing (2019-12-10). Before the @parameterized.named_parameters
# decorator was added, this non-cached session context was set outside
# the iteration loops for the parameter combinations, and so was re-used.
seed = (
hash(data_layout) % 256 + hash(data_rank) % 256 +
hash(data_type) % 256)
np.random.seed(seed)
batch_size = 10
channel_count = 8
data_dim = 14
input_shape = self._makeShapeTuple(batch_size, channel_count, data_rank,
data_dim, data_layout)
bias_shape = (channel_count,)
output_shape = input_shape
input_val = self._randomDataOp(input_shape, data_type)
bias_val = self._randomDataOp(bias_shape, data_type)
data_format = self._dataFormatFromDataLayout(data_layout)
repeat_count = 5
if context.executing_eagerly():
def bias_gradients(local_seed):
np.random.seed(local_seed)
upstream_gradients = self._randomDataOp(output_shape, data_type)
with backprop.GradientTape(persistent=True) as tape:
tape.watch(bias_val)
bias_add_output = nn_ops.bias_add(
input_val, bias_val, data_format=data_format)
gradient_injector_output = bias_add_output * upstream_gradients
return tape.gradient(gradient_injector_output, bias_val)
for i in range(repeat_count):
local_seed = seed + i # select different upstream gradients
result_a = bias_gradients(local_seed)
result_b = bias_gradients(local_seed)
self.assertAllEqual(result_a, result_b)
else: # graph mode
upstream_gradients = array_ops.placeholder(
data_type, shape=output_shape, name='upstream_gradients')
bias_add_output = nn_ops.bias_add(
input_val, bias_val, data_format=data_format)
gradient_injector_output = bias_add_output * upstream_gradients
# The gradient function behaves as if grad_ys is multiplied by the op
# gradient result, not passing the upstram gradients through the op's
# gradient generation graph. This is the reason for using the
# gradient injector
bias_gradients = gradients_impl.gradients(
gradient_injector_output,
bias_val,
grad_ys=None,
colocate_gradients_with_ops=True)[0]
for i in range(repeat_count):
feed_dict = {upstream_gradients: self._randomNDArray(output_shape)}
result_a = bias_gradients.eval(feed_dict=feed_dict)
result_b = bias_gradients.eval(feed_dict=feed_dict)
self.assertAllEqual(result_a, result_b)
# TODO(duncanriach): Re-enable the following three tests for the error checks
# after deterministic functionality is implemented at the CUDA kernel level.
def testInputDims(self):
pass
def testBiasVec(self):
pass
def testBiasInputsMatch(self):
pass
if __name__ == '__main__':
# TODO(reedwm): Merge this file with bias_op_base.py and bias_op_test.py
config.enable_op_determinism()
test.main()
tensorflow/python/kernel_tests/nn_ops/bias_op_test.py
+23
View File
@@ -0,0 +1,23 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Functional tests for BiasAdd."""
from tensorflow.python.kernel_tests.nn_ops import bias_op_base
from tensorflow.python.platform import test
BiasAddTest = bias_op_base.BiasAddTestBase
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/conv1d_test.py
+122
View File
@@ -0,0 +1,122 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for convolution related functionality in tensorflow.ops.nn."""
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import array_ops_stack
from tensorflow.python.ops import nn_ops
from tensorflow.python.platform import test
class Conv1DTest(test.TestCase):
def testBasic(self):
"""Test that argument passing to conv1d is handled properly."""
# double datatype is currently not supported for convolution ops
# on the ROCm platform
optional_float64 = [] if test.is_built_with_rocm() else [dtypes.float64]
for dtype in [dtypes.float16, dtypes.float32] + optional_float64:
x = constant_op.constant([1, 2, 3, 4], dtype=dtype)
x = array_ops.expand_dims(x, 0) # Add batch dimension
x = array_ops.expand_dims(x, 2) # And depth dimension
filters = constant_op.constant([2, 1], dtype=dtype)
filters = array_ops.expand_dims(filters, 1) # in_channels
filters = array_ops.expand_dims(filters, 2) # out_channels
# Filters is 2x1x1
for stride in [1, 2]:
with self.cached_session(use_gpu=test.is_gpu_available()):
c = nn_ops.conv1d(x, filters, stride, padding="VALID")
reduced = array_ops.squeeze(c)
output = self.evaluate(reduced)
if stride == 1:
self.assertEqual(len(output), 3)
self.assertAllClose(output,
[2 * 1 + 1 * 2, 2 * 2 + 1 * 3, 2 * 3 + 1 * 4])
else:
self.assertEqual(len(output), 2)
self.assertAllClose(output, [2 * 1 + 1 * 2, 2 * 3 + 1 * 4])
def testExpandedBatch(self):
"""Test that argument passing to conv1d is handled properly."""
# double datatype is currently not supported for convolution ops
# on the ROCm platform
x = constant_op.constant([1, 2, 3, 4], dtype=dtypes.float32)
x = array_ops.expand_dims(x, 0) # Add batch dimension
x = array_ops.expand_dims(x, 2) # And depth dimension
x = array_ops_stack.stack([x, x]) # Make batch shape [2, 1]
filters = constant_op.constant([2, 1], dtype=dtypes.float32)
filters = array_ops.expand_dims(filters, 1) # in_channels
filters = array_ops.expand_dims(filters, 2) # out_channels
# Filters is 2x1x1
for stride in [1, 2]:
with self.cached_session(use_gpu=test.is_gpu_available()):
c = nn_ops.conv1d(x, filters, stride, padding="VALID")
reduced = array_ops.squeeze(c) # Sequeeze out dims 1 and 3.
output = self.evaluate(reduced)
if stride == 1:
self.assertAllClose(output,
[[2 * 1 + 1 * 2, 2 * 2 + 1 * 3, 2 * 3 + 1 * 4],
[2 * 1 + 1 * 2, 2 * 2 + 1 * 3, 2 * 3 + 1 * 4]])
else:
self.assertAllClose(
output,
[[2 * 1 + 1 * 2, 2 * 3 + 1 * 4], [2 * 1 + 1 * 2, 2 * 3 + 1 * 4]])
def testConv1DTranspose(self):
with self.cached_session():
stride = 2
# Input, output: [batch, width, depth]
x_shape = [2, 4, 3]
y_shape = [2, 9, 2]
# Filter: [kernel_width, output_depth, input_depth]
f_shape = [3, 2, 3]
x = constant_op.constant(
1.0, shape=x_shape, name="x", dtype=dtypes.float32)
f = constant_op.constant(
1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
output = nn_ops.conv1d_transpose(
x, f, y_shape, strides=stride, padding="VALID")
value = self.evaluate(output)
cache_values = np.zeros(y_shape, dtype=np.float32)
# The amount of padding added
pad = 1
for n in range(x_shape[0]):
for k in range(f_shape[1]):
for w in range(pad, y_shape[1] - pad):
target = 3.0
# We add a case for locations divisible by the stride.
w_in = w % stride == 0 and w > pad and w < y_shape[1] - 1 - pad
if w_in:
target += 3.0
cache_values[n, w, k] = target
# copy values in the border
cache_values[n, 0, k] = cache_values[n, 1, k]
cache_values[n, -1, k] = cache_values[n, -2, k]
self.assertAllClose(cache_values, value)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/conv1d_transpose_test.py
+256
View File
@@ -0,0 +1,256 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for convolution related functionality in tensorflow.ops.nn."""
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
class Conv1DTransposeTest(test.TestCase):
def testConv1DTransposeSingleStride(self):
with self.cached_session():
strides = [1, 1, 1]
# Input, output: [batch, width, depth]
x_shape = [2, 6, 3]
y_shape = [2, 6, 2]
# Filter: [kernel_width, output_depth, input_depth]
f_shape = [3, 2, 3]
x = constant_op.constant(
1.0, shape=x_shape, name="x", dtype=dtypes.float32)
f = constant_op.constant(
1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
output = nn_ops.conv1d_transpose(
x, f, y_shape, strides=strides, padding="SAME")
value = self.evaluate(output)
for n in range(y_shape[0]):
for w in range(y_shape[1]):
for c in range(y_shape[2]):
target = 2 * 3.0
w_in = w > 0 and w < y_shape[1] - 1
if w_in:
target += 3.0
self.assertAllClose(target, value[n, w, c])
def testConv1DTransposeSame(self):
with self.cached_session():
strides = [1, 2, 1]
# Input, output: [batch, width, depth]
x_shape = [2, 4, 3]
y_shape = [2, 8, 2]
# Filter: [kernel_width, output_depth, input_depth]
f_shape = [3, 2, 3]
x = constant_op.constant(
1.0, shape=x_shape, name="x", dtype=dtypes.float32)
f = constant_op.constant(
1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
output = nn_ops.conv1d_transpose(
x, f, y_shape, strides=strides, padding="SAME")
value = self.evaluate(output)
for n in range(x_shape[0]):
for k in range(f_shape[1]):
for w in range(y_shape[1]):
target = 3.0
# We add a case for locations divisible by the stride.
w_in = w % strides[1] == 0 and w > 0 and w < y_shape[1] - 1
if w_in:
target += 3.0
self.assertAllClose(target, value[n, w, k])
def testConv1DTransposeValid(self):
with self.cached_session():
strides = [1, 2, 1]
# Input, output: [batch, width, depth]
x_shape = [2, 4, 3]
y_shape = [2, 9, 2]
# Filter: [kernel_width, output_depth, input_depth]
f_shape = [3, 2, 3]
x = constant_op.constant(
1.0, shape=x_shape, name="x", dtype=dtypes.float32)
f = constant_op.constant(
1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
output = nn_ops.conv1d_transpose(
x, f, y_shape, strides=strides, padding="VALID")
value = self.evaluate(output)
cache_values = np.zeros(y_shape, dtype=np.float32)
# The amount of padding added
pad = 1
for n in range(x_shape[0]):
for k in range(f_shape[1]):
for w in range(pad, y_shape[1] - pad):
target = 3.0
# We add a case for locations divisible by the stride.
w_in = w % strides[1] == 0 and w > pad and w < y_shape[1] - 1 - pad
if w_in:
target += 3.0
cache_values[n, w, k] = target
# copy values in the border
cache_values[n, 0, k] = cache_values[n, 1, k]
cache_values[n, -1, k] = cache_values[n, -2, k]
cache_values[n, :, k] = cache_values[n, :, k]
self.assertAllClose(cache_values, value)
@test_util.run_deprecated_v1
def testGradient(self):
self.skipTest("b/262851489: Fix nightly build for GPU.")
x_shape = [2, 4, 3]
f_shape = [3, 2, 3]
y_shape = [2, 8, 2]
strides = [1, 2, 1]
np.random.seed(1) # Make it reproducible.
x_val = np.random.random_sample(x_shape).astype(np.float64)
f_val = np.random.random_sample(f_shape).astype(np.float64)
with self.cached_session():
x = constant_op.constant(x_val, name="x", dtype=dtypes.float32)
f = constant_op.constant(f_val, name="f", dtype=dtypes.float32)
output = nn_ops.conv1d_transpose(
x, f, y_shape, strides=strides, padding="SAME")
err = gradient_checker.compute_gradient_error([x, f], [x_shape, f_shape],
output, y_shape)
print("conv1d_transpose gradient err = %g " % err)
err_tolerance = 0.0005
self.assertLess(err, err_tolerance)
def testConv1DTransposeSingleStrideNCW(self):
# `NCW` data format is only supported for CUDA device.
if test.is_gpu_available(cuda_only=True):
with self.session():
strides = [1, 1, 1]
# Input, output: [batch, depth, width]
x_shape = [2, 3, 4]
y_shape = [2, 2, 4]
# Filter: [kernel_width, output_depth, input_depth]
f_shape = [3, 2, 3]
x = constant_op.constant(
1.0, shape=x_shape, name="x", dtype=dtypes.float32)
f = constant_op.constant(
1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
output = nn_ops.conv1d_transpose(
x, f, y_shape, strides=strides, padding="SAME", data_format="NCW")
value = self.evaluate(output)
for n in range(x_shape[0]):
for k in range(f_shape[1]):
for w in range(y_shape[2]):
target = 2 * 3.0
w_in = w > 0 and w < y_shape[2] - 1
if w_in:
target += 3.0
self.assertAllClose(target, value[n, k, w])
def testConv1DTransposeSameNCW(self):
# `NCW` data format is only supported for CUDA device.
if test.is_gpu_available(cuda_only=True):
with self.session():
strides = [1, 1, 2]
# Input, output: [batch, depth, width]
x_shape = [2, 3, 4]
y_shape = [2, 2, 8]
# Filter: [kernel_width, output_depth, input_depth]
f_shape = [3, 2, 3]
x = constant_op.constant(
1.0, shape=x_shape, name="x", dtype=dtypes.float32)
f = constant_op.constant(
1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
output = nn_ops.conv1d_transpose(
x, f, y_shape, strides=strides, padding="SAME", data_format="NCW")
value = self.evaluate(output)
for n in range(x_shape[0]):
for k in range(f_shape[1]):
for w in range(y_shape[2]):
target = 3.0
# We add a case for locations divisible by the stride.
w_in = w % strides[2] == 0 and w > 0 and w < y_shape[2] - 1
if w_in:
target += 3.0
self.assertAllClose(target, value[n, k, w])
def testConv1DTransposeValidNCW(self):
# `NCW` data format is only supported for CUDA device.
if test.is_gpu_available(cuda_only=True):
with self.session():
strides = [1, 1, 2]
# Input, output: [batch, depth, width]
x_shape = [2, 3, 4]
y_shape = [2, 2, 9]
# Filter: [kernel_width, output_depth, input_depth]
f_shape = [3, 2, 3]
x = constant_op.constant(
1.0, shape=x_shape, name="x", dtype=dtypes.float32)
f = constant_op.constant(
1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
output = nn_ops.conv1d_transpose(
x, f, y_shape, strides=strides, padding="VALID", data_format="NCW")
value = self.evaluate(output)
cache_values = np.zeros(y_shape, dtype=np.float32)
# The amount of padding added
pad = 1
for n in range(x_shape[0]):
for k in range(f_shape[1]):
for w in range(pad, y_shape[2] - pad):
target = 3.0
# We add a case for locations divisible by the stride.
w_in = w % strides[2] == 0 and w > pad and \
w < y_shape[2] - 1 - pad
if w_in:
target += 3.0
cache_values[n, k, w] = target
# copy values in the border
cache_values[n, k, 0] = cache_values[n, k, 1]
cache_values[n, k, -1] = cache_values[n, k, -2]
cache_values[n, k, :] = cache_values[n, k, :]
self.assertAllClose(cache_values, value)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/conv2d_backprop_filter_grad_test.py
+122
View File
@@ -0,0 +1,122 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for convolution related functionality in tensorflow.ops.nn."""
import unittest
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
@test_util.run_all_without_tensor_float_32(
"Run Conv2D backprop without TF32 on GPU")
class Conv2DBackpropFilterGradTest(test.TestCase):
# TODO(b/292002914): Enable this test after fixing its flakyness.
@unittest.skip("Disable the flaky test.")
@test_util.run_deprecated_v1
def testGradient(self):
with self.cached_session():
for padding in [
"SAME",
"VALID",
[(0, 0), (1, 2), (3, 4), (0, 0)],
[(0, 0), (0, 3), (4, 2), (0, 0)]
]:
for stride in [1, 2]:
np.random.seed(1)
in_shape = [5, 8, 6, 4]
in_val = constant_op.constant(
2 * np.random.random_sample(in_shape) - 1, dtype=dtypes.float32)
filter_shape = [3, 3, 4, 6]
# Make a convolution op with the current settings, just to easily get
# the shape of the output.
conv_out = nn_ops.conv2d(
in_val,
array_ops.zeros(filter_shape),
strides=[1, stride, stride, 1],
padding=padding)
out_backprop_shape = conv_out.get_shape().as_list()
out_backprop_val = constant_op.constant(
2 * np.random.random_sample(out_backprop_shape) - 1,
dtype=dtypes.float32)
output = nn_ops.conv2d_backprop_filter(
in_val,
filter_shape,
out_backprop_val,
strides=[1, stride, stride, 1],
padding=padding)
err = gradient_checker.compute_gradient_error(
[in_val, out_backprop_val], [in_shape, out_backprop_shape],
output, filter_shape)
print("conv2d_backprop_filter gradient err = %g " % err)
err_tolerance = 3e-2 if test.is_gpu_available() else 2e-3
self.assertLess(
err,
err_tolerance,
msg="padding={0},stride={1},".format(str(padding), stride))
@test_util.run_deprecated_v1
def testGradientDilatedConv(self):
if test.is_gpu_available(cuda_only=True):
with self.session():
for padding in [
"SAME",
"VALID",
[(0, 0), (3, 5), (2, 1), (0, 0)],
[(0, 0), (5, 2), (5, 1), (0, 0)]
]:
for stride in [1, 2]:
np.random.seed(1)
in_shape = [5, 8, 6, 4]
in_val = constant_op.constant(
2 * np.random.random_sample(in_shape) - 1, dtype=dtypes.float32)
filter_shape = [3, 3, 4, 6]
# Make a convolution op with the current settings,
# just to easily get the shape of the output.
conv_out = nn_ops.conv2d(
in_val,
array_ops.zeros(filter_shape),
dilations=[1, 2, 2, 1],
strides=[1, stride, stride, 1],
padding=padding)
out_backprop_shape = conv_out.get_shape().as_list()
out_backprop_val = constant_op.constant(
2 * np.random.random_sample(out_backprop_shape) - 1,
dtype=dtypes.float32)
output = nn_ops.conv2d_backprop_filter(
in_val,
filter_shape,
out_backprop_val,
dilations=[1, 2, 2, 1],
strides=[1, stride, stride, 1],
padding=padding)
err = gradient_checker.compute_gradient_error(
[in_val, out_backprop_val], [in_shape, out_backprop_shape],
output, filter_shape)
print("conv2d_backprop_filter gradient err = %g " % err)
err_tolerance = 1e-2
self.assertLess(err, err_tolerance)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/conv2d_transpose_test.py
+336
View File
@@ -0,0 +1,336 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for convolution related functionality in tensorflow.ops.nn."""
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import array_ops_stack
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import nn_ops
from tensorflow.python.ops import random_ops
from tensorflow.python.ops import variable_scope
from tensorflow.python.ops import variables
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
class Conv2DTransposeTest(test.TestCase):
def testConv2DTransposeSingleStride(self):
with self.cached_session():
for dtype in (dtypes.float32, dtypes.int32):
strides = [1, 1, 1, 1]
# Input, output: [batch, height, width, depth]
x_shape = [2, 6, 4, 3]
y_shape = [2, 6, 4, 2]
# Filter: [kernel_height, kernel_width, output_depth, input_depth]
f_shape = [3, 3, 2, 3]
x = constant_op.constant(1, shape=x_shape, name="x", dtype=dtype)
f = constant_op.constant(1, shape=f_shape, name="filter", dtype=dtype)
output = nn_ops.conv2d_transpose(
x, f, y_shape, strides=strides, padding="SAME")
value = self.evaluate(output)
# We count the number of cells being added at the locations in the
# output.
# At the center, #cells=kernel_height * kernel_width
# At the corners, #cells=ceil(kernel_height/2) * ceil(kernel_width/2)
# At the borders, #cells=ceil(kernel_height/2)*kernel_width or
# kernel_height * ceil(kernel_width/2)
for n in range(x_shape[0]):
for k in range(f_shape[2]):
for w in range(y_shape[2]):
for h in range(y_shape[1]):
target = 4 * 3
h_in = h > 0 and h < y_shape[1] - 1
w_in = w > 0 and w < y_shape[2] - 1
if h_in and w_in:
target += 5 * 3
elif h_in or w_in:
target += 2 * 3
if dtype.is_integer:
self.assertAllEqual(target, value[n, h, w, k])
else:
self.assertAllClose(target, value[n, h, w, k])
def testConv2DTransposeSame(self):
with self.cached_session():
for dtype in (dtypes.float32, dtypes.int32):
strides = [1, 2, 2, 1]
# Input, output: [batch, height, width, depth]
x_shape = [2, 6, 4, 3]
y_shape = [2, 12, 8, 2]
# Filter: [kernel_height, kernel_width, output_depth, input_depth]
f_shape = [3, 3, 2, 3]
x = constant_op.constant(1, shape=x_shape, name="x", dtype=dtype)
f = constant_op.constant(1, shape=f_shape, name="filter", dtype=dtype)
output = nn_ops.conv2d_transpose(
x, f, y_shape, strides=strides, padding="SAME")
value = self.evaluate(output)
for n in range(x_shape[0]):
for k in range(f_shape[2]):
for w in range(y_shape[2]):
for h in range(y_shape[1]):
target = 3
# We add a case for locations divisible by the stride.
h_in = h % strides[1] == 0 and h > 0 and h < y_shape[1] - 1
w_in = w % strides[2] == 0 and w > 0 and w < y_shape[2] - 1
if h_in and w_in:
target += 9
elif h_in or w_in:
target += 3
if dtype.is_integer:
self.assertAllEqual(target, value[n, h, w, k])
else:
self.assertAllClose(target, value[n, h, w, k])
def testConv2DTransposeValid(self):
with self.cached_session():
for dtype in (dtypes.float32, dtypes.int32):
strides = [1, 2, 2, 1]
# Input, output: [batch, height, width, depth]
x_shape = [2, 6, 4, 3]
y_shape = [2, 13, 9, 2]
# Filter: [kernel_height, kernel_width, output_depth, input_depth]
f_shape = [3, 3, 2, 3]
x = constant_op.constant(1, shape=x_shape, name="x", dtype=dtype)
f = constant_op.constant(1, shape=f_shape, name="filter", dtype=dtype)
output = nn_ops.conv2d_transpose(
x, f, y_shape, strides=strides, padding="VALID")
value = self.evaluate(output)
cache_values = np.zeros(y_shape, dtype=np.float32)
# The amount of padding added
pad = 1
for n in range(x_shape[0]):
for k in range(f_shape[2]):
for w in range(pad, y_shape[2] - pad):
for h in range(pad, y_shape[1] - pad):
target = 3
# We add a case for locations divisible by the stride.
h_in = h % strides[1] == 0 and h > pad and h < y_shape[
1] - 1 - pad
w_in = w % strides[2] == 0 and w > pad and w < y_shape[
2] - 1 - pad
if h_in and w_in:
target += 9
elif h_in or w_in:
target += 3
cache_values[n, h, w, k] = target
# copy values in the border
cache_values[n, :, 0, k] = cache_values[n, :, 1, k]
cache_values[n, :, -1, k] = cache_values[n, :, -2, k]
cache_values[n, 0, :, k] = cache_values[n, 1, :, k]
cache_values[n, -1, :, k] = cache_values[n, -2, :, k]
if dtype.is_integer:
self.assertAllEqual(cache_values, value)
else:
self.assertAllClose(cache_values, value)
@test_util.run_deprecated_v1
def testGradient(self):
self.skipTest("b/262851489: Fix nightly build for GPU.")
x_shape = [2, 6, 4, 3]
f_shape = [3, 3, 2, 3]
y_shape = [2, 12, 8, 2]
strides = [1, 2, 2, 1]
np.random.seed(1) # Make it reproducible.
x_val = np.random.random_sample(x_shape).astype(np.float64)
f_val = np.random.random_sample(f_shape).astype(np.float64)
with self.cached_session():
x = constant_op.constant(x_val, name="x", dtype=dtypes.float32)
f = constant_op.constant(f_val, name="f", dtype=dtypes.float32)
output = nn_ops.conv2d_transpose(
x, f, y_shape, strides=strides, padding="SAME")
err = gradient_checker.compute_gradient_error([x, f], [x_shape, f_shape],
output, y_shape)
print("conv2d_transpose gradient err = %g " % err)
err_tolerance = 0.0006
self.assertLess(err, err_tolerance)
def testConv2DTransposeSingleStrideNCHW(self):
# `NCHW` data format is only supported for CUDA device.
if test.is_gpu_available(cuda_only=True):
with self.session():
strides = [1, 1, 1, 1]
# Input, output: [batch, depth, height, width, depth]
x_shape = [2, 3, 6, 4]
y_shape = [2, 2, 6, 4]
# Filter: [kernel_height, kernel_width, output_depth, input_depth]
f_shape = [3, 3, 2, 3]
x = constant_op.constant(
1.0, shape=x_shape, name="x", dtype=dtypes.float32)
f = constant_op.constant(
1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
output = nn_ops.conv2d_transpose(
x, f, y_shape, strides=strides, padding="SAME", data_format="NCHW")
value = self.evaluate(output)
for n in range(x_shape[0]):
for k in range(f_shape[2]):
for w in range(y_shape[3]):
for h in range(y_shape[2]):
target = 4 * 3.0
h_in = h > 0 and h < y_shape[2] - 1
w_in = w > 0 and w < y_shape[3] - 1
if h_in and w_in:
target += 5 * 3.0
elif h_in or w_in:
target += 2 * 3.0
self.assertAllClose(target, value[n, k, h, w])
def testConv2DTransposeSameNCHW(self):
# `NCHW` data format is only supported for CUDA device.
if test.is_gpu_available(cuda_only=True):
with self.session():
strides = [1, 1, 2, 2]
# Input, output: [batch, depth, height, width]
x_shape = [2, 3, 6, 4]
y_shape = [2, 2, 12, 8]
# Filter: [kernel_height, kernel_width, output_depth, input_depth]
f_shape = [3, 3, 2, 3]
x = constant_op.constant(
1.0, shape=x_shape, name="x", dtype=dtypes.float32)
f = constant_op.constant(
1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
output = nn_ops.conv2d_transpose(
x, f, y_shape, strides=strides, padding="SAME", data_format="NCHW")
value = self.evaluate(output)
for n in range(x_shape[0]):
for k in range(f_shape[2]):
for w in range(y_shape[3]):
for h in range(y_shape[2]):
target = 3.0
# We add a case for locations divisible by the stride.
h_in = h % strides[2] == 0 and h > 0 and h < y_shape[2] - 1
w_in = w % strides[3] == 0 and w > 0 and w < y_shape[3] - 1
if h_in and w_in:
target += 9.0
elif h_in or w_in:
target += 3.0
self.assertAllClose(target, value[n, k, h, w])
def testConv2DTransposeValidNCHW(self):
# `NCHW` data format is only supported for CUDA device.
if test.is_gpu_available(cuda_only=True):
with self.session():
strides = [1, 1, 2, 2]
# Input, output: [batch, depth, height, width]
x_shape = [2, 3, 6, 4]
y_shape = [2, 2, 13, 9]
# Filter: [kernel_height, kernel_width, output_depth, input_depth]
f_shape = [3, 3, 2, 3]
x = constant_op.constant(
1.0, shape=x_shape, name="x", dtype=dtypes.float32)
f = constant_op.constant(
1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
output = nn_ops.conv2d_transpose(
x, f, y_shape, strides=strides, padding="VALID", data_format="NCHW")
value = self.evaluate(output)
cache_values = np.zeros(y_shape, dtype=np.float32)
# The amount of padding added
pad = 1
for n in range(x_shape[0]):
for k in range(f_shape[2]):
for w in range(pad, y_shape[3] - pad):
for h in range(pad, y_shape[2] - pad):
target = 3.0
# We add a case for locations divisible by the stride.
h_in = h % strides[2] == 0 and h > pad and h < y_shape[
2] - 1 - pad
w_in = w % strides[3] == 0 and w > pad and w < y_shape[
3] - 1 - pad
if h_in and w_in:
target += 9.0
elif h_in or w_in:
target += 3.0
cache_values[n, k, h, w] = target
# copy values in the border
cache_values[n, k, :, 0] = cache_values[n, k, :, 1]
cache_values[n, k, :, -1] = cache_values[n, k, :, -2]
cache_values[n, k, 0, :] = cache_values[n, k, 1, :]
cache_values[n, k, -1, :] = cache_values[n, k, -2, :]
self.assertAllClose(cache_values, value)
def testConv2DTransposeShapeInference(self):
# Test case for 8972
initializer = random_ops.truncated_normal(
[3, 3, 5, 1], mean=0.0, stddev=0.01, dtype=dtypes.float32)
x = variables.Variable(random_ops.random_normal([3, 10, 5, 1]))
f = variable_scope.get_variable("f", initializer=initializer)
f_shape = array_ops_stack.stack([array_ops.shape(x)[0], 10, 5, 5])
output = nn_ops.conv2d_transpose(
x, f, f_shape, strides=[1, 1, 1, 1], padding="SAME")
self.assertEqual(output.get_shape().as_list(), [3, 10, 5, 5])
def testConv2DTransposeInvalidOutputShape(self):
with self.session():
with self.assertRaises((errors.InvalidArgumentError, ValueError)):
op = nn_ops.conv2d_transpose(
input=np.ones((1, 1, 1, 1)),
filters=np.ones((1, 1, 1, 1)),
output_shape=[2, -2],
strides=[1])
self.evaluate(op)
def testConv2DTransposeLargeOutputShape(self):
# On GPU, this test does try to allocate the output tensor and OOMs.
with test_util.device(use_gpu=False):
with self.assertRaises((errors.InvalidArgumentError, ValueError)):
op = nn_ops.conv2d_transpose(
input=np.ones((2, 2, 2, 2)),
output_shape=[114078056, 179835296],
strides=[10],
filters=[[[[1]]]])
self.evaluate(op)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/conv3d_backprop_filter_v2_grad_test.py
+81
View File
@@ -0,0 +1,81 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for convolution related functionality in tensorflow.ops.nn."""
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
class Conv3DBackpropFilterV2GradTest(test.TestCase):
@test_util.run_deprecated_v1
def testGradient(self):
with self.cached_session():
for padding in ["SAME", "VALID"]:
for stride in [1, 2]:
np.random.seed(1)
in_shape = [2, 4, 3, 3, 2]
in_val = constant_op.constant(
2 * np.random.random_sample(in_shape) - 1, dtype=dtypes.float32)
filter_shape = [3, 3, 3, 2, 3]
strides = [1, stride, stride, stride, 1]
# Make a convolution op with the current settings, just to easily get
# the shape of the output.
conv_out = nn_ops.conv3d(in_val,
array_ops.zeros(filter_shape), strides,
padding)
out_backprop_shape = conv_out.get_shape().as_list()
out_backprop_val = constant_op.constant(
2 * np.random.random_sample(out_backprop_shape) - 1,
dtype=dtypes.float32)
output = nn_ops.conv3d_backprop_filter_v2(in_val, filter_shape,
out_backprop_val, strides,
padding)
err = gradient_checker.compute_gradient_error(
[in_val, out_backprop_val], [in_shape, out_backprop_shape],
output, filter_shape)
print("conv3d_backprop_filter gradient err = %g " % err)
err_tolerance = 1e-3
self.assertLess(err, err_tolerance)
def testBadFilterShape(self):
strides = [1, 1, 1, 1, 1]
padding = "VALID"
tin = constant_op.constant(
.5053710941, shape=[2, 2, 2, 2, 1], dtype=dtypes.float32)
filter_sizes = constant_op.constant(0, shape=[], dtype=dtypes.int32)
out_backprop = constant_op.constant(
.5053710941, shape=[2, 2, 2, 2, 1], dtype=dtypes.float32)
with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError),
"must be rank 1"):
nn_ops.conv3d_backprop_filter_v2(
input=tin,
filter_sizes=filter_sizes,
out_backprop=out_backprop,
strides=strides,
padding=padding)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/conv3d_transpose_test.py
+238
View File
@@ -0,0 +1,238 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for convolution related functionality in tensorflow.ops.nn."""
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
class Conv3DTransposeTest(test.TestCase):
def testConv3DTransposeSingleStride(self):
with self.cached_session():
strides = [1, 1, 1, 1, 1]
# Input, output: [batch, depth, height, width, channel]
x_shape = [2, 5, 6, 4, 3]
y_shape = [2, 5, 6, 4, 2]
# Filter: [kernel_depth, kernel_height, kernel_width, out_depth, in_depth]
f_shape = [3, 3, 3, 2, 3]
x = constant_op.constant(
1.0, shape=x_shape, name="x", dtype=dtypes.float32)
f = constant_op.constant(
1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
output = nn_ops.conv3d_transpose(
x, f, y_shape, strides=strides, padding="SAME")
value = self.evaluate(output)
# We count the number of cells being added at the locations in the output.
# At the center, #cells = kernel_depth * kernel_height * kernel_width
# At the corners, #cells = ceil(kernel_depth/2) * ceil(kernel_height/2)
# * ceil(kernel_width/2)
# At the edges, #cells =
# kernel_depth * ceil(kernel_height/2) * ceil(kernel_width/2) or
# ceil(kernel_depth/2) * kernel_height * ceil(kernel_width/2) or
# ceil(kernel_depth/2) * ceil(kernel_height/2) * kernel_width
# At the borders, #cells =
# ceil(kernel_depth/2) * kernel_height * kernel_width or
# kernel_depth * ceil(kernel_height/2) * kernel_width or
# kernel_depth * kernel_height * ceil(kernel_width/2)
for n in range(x_shape[0]):
for k in range(f_shape[3]):
for w in range(y_shape[3]):
for h in range(y_shape[2]):
for d in range(y_shape[1]):
d_in = d > 0 and d < y_shape[1] - 1
h_in = h > 0 and h < y_shape[2] - 1
w_in = w > 0 and w < y_shape[3] - 1
if d_in + h_in + w_in == 3:
target = 27 * 3.0
elif d_in + h_in + w_in == 2:
target = 18 * 3.0
elif d_in or h_in or w_in:
target = 12 * 3.0
else:
target = 8 * 3.0
self.assertAllClose(target, value[n, d, h, w, k])
def testConv3DTransposeSame(self):
with self.cached_session():
strides = [1, 2, 2, 2, 1]
# Input, output: [batch, depth, height, width, depth]
x_shape = [2, 5, 6, 4, 3]
y_shape = [2, 10, 12, 8, 2]
# Filter: [kernel_depth, kernel_height, kernel_width, out_depth, in_depth]
f_shape = [3, 3, 3, 2, 3]
x = constant_op.constant(
1.0, shape=x_shape, name="x", dtype=dtypes.float32)
f = constant_op.constant(
1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
output = nn_ops.conv3d_transpose(
x, f, y_shape, strides=strides, padding="SAME")
value = self.evaluate(output)
for n in range(x_shape[0]):
for k in range(f_shape[3]):
for w in range(y_shape[3]):
for h in range(y_shape[2]):
for d in range(y_shape[1]):
# We add a case for locations divisible by the stride.
d_in = d % strides[1] == 0 and 0 < d < y_shape[1] - 1
h_in = h % strides[2] == 0 and 0 < h < y_shape[2] - 1
w_in = w % strides[3] == 0 and 0 < w < y_shape[3] - 1
if d_in + h_in + w_in == 3:
target = 8 * 3.0
elif d_in + h_in + w_in == 2:
target = 4 * 3.0
elif d_in or h_in or w_in:
target = 2 * 3.0
else:
target = 3.0
self.assertAllClose(target, value[n, d, h, w, k])
@test_util.run_deprecated_v1
def testConv3DTransposeShapeMismatch(self):
# Test case for GitHub issue 18460
x_shape = [2, 2, 3, 4, 3]
f_shape = [3, 3, 3, 2, 2]
y_shape = [2, 2, 6, 8, 6]
strides = [1, 1, 2, 2, 2]
np.random.seed(1)
x_value = np.random.random_sample(x_shape).astype(np.float64)
f_value = np.random.random_sample(f_shape).astype(np.float64)
nn_ops.conv3d_transpose(
x_value, f_value, y_shape, strides, data_format="NCDHW")
def testConv3DTransposeOutputShapeType(self):
# Test case for GitHub issue 18887
for dtype in [dtypes.int32, dtypes.int64]:
with self.cached_session():
x_shape = [2, 5, 6, 4, 3]
y_shape = [2, 5, 6, 4, 2]
f_shape = [3, 3, 3, 2, 3]
strides = [1, 1, 1, 1, 1]
x_value = constant_op.constant(
1.0, shape=x_shape, name="x", dtype=dtypes.float32)
f_value = constant_op.constant(
1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
output = nn_ops.conv3d_transpose(
x_value, f_value, constant_op.constant(y_shape, dtype=dtype),
strides=strides, padding="SAME")
self.evaluate(output)
def testConv3DTransposeValid(self):
with self.cached_session():
strides = [1, 2, 2, 2, 1]
# Input, output: [batch, depth, height, width, depth]
x_shape = [2, 5, 6, 4, 3]
y_shape = [2, 11, 13, 9, 2]
# Filter: [kernel_depth, kernel_height, kernel_width, out_depth, in_depth]
f_shape = [3, 3, 3, 2, 3]
x = constant_op.constant(
1.0, shape=x_shape, name="x", dtype=dtypes.float32)
f = constant_op.constant(
1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
output = nn_ops.conv3d_transpose(
x, f, y_shape, strides=strides, padding="VALID")
value = self.evaluate(output)
cache_values = np.zeros(y_shape, dtype=np.float32)
# The amount of padding added
pad = 1
for n in range(x_shape[0]):
for k in range(f_shape[3]):
for w in range(y_shape[3]):
for h in range(y_shape[2]):
for d in range(y_shape[1]):
# We add a case for locations divisible by the stride.
d_in = d % strides[1] == 0 and pad < d < y_shape[1] - 1 - pad
h_in = h % strides[2] == 0 and pad < h < y_shape[2] - 1 - pad
w_in = w % strides[3] == 0 and pad < w < y_shape[3] - 1 - pad
if d_in + h_in + w_in == 3:
target = 8 * 3.0
elif d_in + h_in + w_in == 2:
target = 4 * 3.0
elif d_in or h_in or w_in:
target = 2 * 3.0
else:
target = 3.0
cache_values[n, d, h, w, k] = target
# copy values in the border
cache_values[n, :, :, 0, k] = cache_values[n, :, :, 1, k]
cache_values[n, :, :, -1, k] = cache_values[n, :, :, -2, k]
cache_values[n, :, 0, :, k] = cache_values[n, :, 1, :, k]
cache_values[n, :, -1, :, k] = cache_values[n, :, -2, :, k]
cache_values[n, 0, :, :, k] = cache_values[n, 1, :, :, k]
cache_values[n, -1, :, :, k] = cache_values[n, -2, :, :, k]
self.assertAllClose(cache_values, value)
@test_util.run_deprecated_v1
def testGradient(self):
self.skipTest("b/262851489: Fix nightly build for GPU.")
x_shape = [2, 3, 4, 3, 2]
f_shape = [3, 3, 3, 2, 2]
y_shape = [2, 6, 8, 6, 2]
strides = [1, 2, 2, 2, 1]
np.random.seed(1) # Make it reproducible.
x_val = np.random.random_sample(x_shape).astype(np.float64)
f_val = np.random.random_sample(f_shape).astype(np.float64)
with self.cached_session():
x = constant_op.constant(x_val, name="x", dtype=dtypes.float32)
f = constant_op.constant(f_val, name="f", dtype=dtypes.float32)
output = nn_ops.conv3d_transpose(
x, f, y_shape, strides=strides, padding="SAME")
err = gradient_checker.compute_gradient_error([x, f], [x_shape, f_shape],
output, y_shape)
print("conv3d_transpose gradient err = %g " % err)
err_tolerance = 0.00055
self.assertLess(err, err_tolerance)
def testConv3DTransposeZeroShapeDoNotRaiseError(self):
with self.cached_session():
x_value = np.zeros([10, 0, 2, 3, 3])
f_value = np.ones((3, 3, 3, 3, 3))
y_shape = np.stack([10, 0, 2, 3, 3])
output = nn_ops.conv3d_transpose(
x_value,
f_value,
y_shape,
strides=(1, 1, 1),
data_format="NDHWC",
padding="SAME",
)
_ = self.evaluate(output)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/conv_ops_3d_test.py
+1116
View File
File diff suppressed because it is too large Load Diff
tensorflow/python/kernel_tests/nn_ops/conv_ops_test.py
+4054
View File
File diff suppressed because it is too large Load Diff
tensorflow/python/kernel_tests/nn_ops/ctc_decoder_ops_test.py
+273
View File
@@ -0,0 +1,273 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tensorflow.ctc_ops.ctc_decoder_ops."""
import itertools
import numpy as np
from tensorflow.python.framework import errors
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops_stack
from tensorflow.python.ops import ctc_ops
from tensorflow.python.platform import test
def grouper(iterable, n, fillvalue=None):
"""Collect data into fixed-length chunks or blocks."""
# grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
args = [iter(iterable)] * n
return itertools.zip_longest(fillvalue=fillvalue, *args)
def flatten(list_of_lists):
"""Flatten one level of nesting."""
return itertools.chain.from_iterable(list_of_lists)
class CTCGreedyDecoderTest(test.TestCase):
def _testCTCDecoder(self,
decoder,
inputs,
seq_lens,
log_prob_truth,
decode_truth,
expected_err_re=None,
**decoder_args):
inputs_t = [ops.convert_to_tensor(x) for x in inputs]
# convert inputs_t into a [max_time x batch_size x depth] tensor
# from a len time python list of [batch_size x depth] tensors
inputs_t = array_ops_stack.stack(inputs_t)
with self.cached_session(use_gpu=False) as sess:
decoded_list, log_probability = decoder(
inputs_t, sequence_length=seq_lens, **decoder_args)
decoded_unwrapped = list(
flatten([(st.indices, st.values, st.dense_shape) for st in
decoded_list]))
if expected_err_re is None:
outputs = sess.run(decoded_unwrapped + [log_probability])
# Group outputs into (ix, vals, shape) tuples
output_sparse_tensors = list(grouper(outputs[:-1], 3))
output_log_probability = outputs[-1]
# Check the number of decoded outputs (top_paths) match
self.assertEqual(len(output_sparse_tensors), len(decode_truth))
# For each SparseTensor tuple, compare (ix, vals, shape)
for out_st, truth_st, tf_st in zip(output_sparse_tensors, decode_truth,
decoded_list):
self.assertAllEqual(out_st[0], truth_st[0]) # ix
self.assertAllEqual(out_st[1], truth_st[1]) # vals
self.assertAllEqual(out_st[2], truth_st[2]) # shape
# Compare the shapes of the components with the truth. The
# `None` elements are not known statically.
self.assertEqual([None, truth_st[0].shape[1]],
tf_st.indices.get_shape().as_list())
self.assertEqual([None], tf_st.values.get_shape().as_list())
self.assertShapeEqual(truth_st[2], tf_st.dense_shape)
# Make sure decoded probabilities match
self.assertAllClose(output_log_probability, log_prob_truth, atol=1e-6)
else:
with self.assertRaisesOpError(expected_err_re):
sess.run(decoded_unwrapped + [log_probability])
@test_util.run_deprecated_v1
def testCTCGreedyDecoder(self):
"""Test two batch entries - best path decoder."""
max_time_steps = 6
# depth == 4
seq_len_0 = 4
input_prob_matrix_0 = np.asarray(
[
[1.0, 0.0, 0.0, 0.0], # t=0
[0.0, 0.0, 0.4, 0.6], # t=1
[0.0, 0.0, 0.4, 0.6], # t=2
[0.0, 0.9, 0.1, 0.0], # t=3
[0.0, 0.0, 0.0, 0.0], # t=4 (ignored)
[0.0, 0.0, 0.0, 0.0]
], # t=5 (ignored)
dtype=np.float32)
input_log_prob_matrix_0 = np.log(input_prob_matrix_0)
seq_len_1 = 5
# dimensions are time x depth
input_prob_matrix_1 = np.asarray(
[
[0.1, 0.9, 0.0, 0.0], # t=0
[0.0, 0.9, 0.1, 0.0], # t=1
[0.0, 0.0, 0.1, 0.9], # t=2
[0.0, 0.9, 0.1, 0.1], # t=3
[0.9, 0.1, 0.0, 0.0], # t=4
[0.0, 0.0, 0.0, 0.0] # t=5 (ignored)
],
dtype=np.float32)
input_log_prob_matrix_1 = np.log(input_prob_matrix_1)
# len max_time_steps array of batch_size x depth matrices
inputs = np.array([
np.vstack(
[input_log_prob_matrix_0[t, :], input_log_prob_matrix_1[t, :]])
for t in range(max_time_steps)
])
# batch_size length vector of sequence_lengths
seq_lens = np.array([seq_len_0, seq_len_1], dtype=np.int32)
# batch_size length vector of negative log probabilities
log_prob_truth = np.array([
np.sum(-np.log([1.0, 0.6, 0.6, 0.9])),
np.sum(-np.log([0.9, 0.9, 0.9, 0.9, 0.9]))
], np.float32)[:, np.newaxis]
# decode_truth: one SparseTensor (ix, vals, shape)
decode_truth = [
(
np.array(
[
[0, 0], # batch 0, 2 outputs
[0, 1],
[1, 0], # batch 1, 3 outputs
[1, 1],
[1, 2]
],
dtype=np.int64),
np.array(
[
0, # batch 0, 2 values
1,
1, # batch 1, 3 values
1,
0
],
dtype=np.int64),
# shape is batch x max_decoded_length
np.array([2, 3], dtype=np.int64)),
]
# Test without defining blank_index
self._testCTCDecoder(ctc_ops.ctc_greedy_decoder, inputs, seq_lens,
log_prob_truth, decode_truth)
# Shift blank_index to be somewhere in the middle of inputs
blank_index = 2
inputs = np.concatenate(
(inputs[:, :, :blank_index], inputs[:, :, -1:], inputs[:, :,
blank_index:-1]),
axis=2)
# Test positive value in blank_index
self._testCTCDecoder(
ctc_ops.ctc_greedy_decoder,
inputs,
seq_lens,
log_prob_truth,
decode_truth,
blank_index=2)
# Test negative value in blank_index
self._testCTCDecoder(
ctc_ops.ctc_greedy_decoder,
inputs,
seq_lens,
log_prob_truth,
decode_truth,
blank_index=-2)
@test_util.run_deprecated_v1
def testCTCDecoderBeamSearch(self):
"""Test one batch, two beams - hibernating beam search."""
# max_time_steps == 8
depth = 6
seq_len_0 = 5
input_prob_matrix_0 = np.asarray(
[
[0.30999, 0.309938, 0.0679938, 0.0673362, 0.0708352, 0.173908],
[0.215136, 0.439699, 0.0370931, 0.0393967, 0.0381581, 0.230517],
[0.199959, 0.489485, 0.0233221, 0.0251417, 0.0233289, 0.238763],
[0.279611, 0.452966, 0.0204795, 0.0209126, 0.0194803, 0.20655],
[0.51286, 0.288951, 0.0243026, 0.0220788, 0.0219297, 0.129878],
# Random entry added in at time=5
[0.155251, 0.164444, 0.173517, 0.176138, 0.169979, 0.160671]
],
dtype=np.float32)
# Add arbitrary offset - this is fine
input_prob_matrix_0 = input_prob_matrix_0 + 2.0
# len max_time_steps array of batch_size x depth matrices
inputs = ([
input_prob_matrix_0[t, :][np.newaxis, :] for t in range(seq_len_0)
] # Pad to max_time_steps = 8
+ 2 * [np.zeros(
(1, depth), dtype=np.float32)])
# batch_size length vector of sequence_lengths
seq_lens = np.array([seq_len_0], dtype=np.int32)
# batch_size length vector of log probabilities
log_prob_truth = np.array(
[
-5.811451, # output beam 0
-6.63339 # output beam 1
],
np.float32)[np.newaxis, :]
# decode_truth: two SparseTensors, (ix, values, shape)
decode_truth = [
# beam 0, batch 0, two outputs decoded
(np.array(
[[0, 0], [0, 1]], dtype=np.int64), np.array(
[1, 0], dtype=np.int64), np.array(
[1, 2], dtype=np.int64)),
# beam 1, batch 0, one output decoded
(np.array(
[[0, 0]], dtype=np.int64), np.array(
[1], dtype=np.int64), np.array(
[1, 1], dtype=np.int64)),
]
# Test correct decoding.
self._testCTCDecoder(
ctc_ops.ctc_beam_search_decoder,
inputs,
seq_lens,
log_prob_truth,
decode_truth,
beam_width=2,
top_paths=2)
# Requesting more paths than the beam width allows.
with self.assertRaisesRegex(errors.InvalidArgumentError,
(".*requested more paths than the beam "
"width.*")):
self._testCTCDecoder(
ctc_ops.ctc_beam_search_decoder,
inputs,
seq_lens,
log_prob_truth,
decode_truth,
beam_width=2,
top_paths=3)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/ctc_loss_op_test.py
+1240
View File
File diff suppressed because it is too large Load Diff
tensorflow/python/kernel_tests/nn_ops/cudnn_d9m_test.py
+29
View File
@@ -0,0 +1,29 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for TF_CUDNN_DETERMINISTIC=1."""
import os
from tensorflow.python.kernel_tests.nn_ops import cudnn_deterministic_base
from tensorflow.python.platform import test
ConvolutionTest = cudnn_deterministic_base.ConvolutionTest
if __name__ == '__main__':
# Note that the effect of setting the following environment variable to
# 'true' is not tested. Unless we can find a simpler pattern for testing these
# environment variables, it would require another test file to be added.
os.environ['TF_CUDNN_DETERMINISTIC'] = '1'
test.main()
tensorflow/python/kernel_tests/nn_ops/cudnn_deterministic_base.py
+292
View File
@@ -0,0 +1,292 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for deterministic cuDNN functionality."""
import collections
import numpy as np
from tensorflow.python.eager import backprop
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import nn_ops
from tensorflow.python.platform import test
# Notes:
#
# TensorFlow makes cuDNN run deterministically when op determinism is enabled
# via tf.config.experimental.enable_op_determinism(). Additionally, setting the
# environmental variable TF_CUDNN_DETERMINISTIC to 'true' or '1' makes cuDNN run
# deterministically, although this environemtnal variable is deprecated and will
# be removed in a future TensorFlow version. Unlike the enable_op_determinism()
# function, the environmental variable only makes ops using cuDNN deterministic,
# not all TensorFlow ops.
#
# Where both deterministic and non-deterministic cuDNN algorithms are available,
# selecting determinitic operation will lead to only the deterministic
# algorithms being chosen. Additionally, selecting deterministic operation will
# result in a deterministic, or reproducible, selection of algorithms (for any
# given layer configuration) for each of the forward and the two backward paths.
#
# These tests intend to confirm that deterministic algorithms are chosen (for
# the back-prop paths) when desterministic operation is selected. The tested
# configurations were first confirmed to produce non-deterministic results when
# the above-mentioned environment variables are not set.
#
# Even though selecting determinitic operation should ensure that the same
# algorithms, for a given layer configuration, are always used (i.e. that
# algorithm selection is deterministic / reproducible), this is not tested.
# TODO(duncanriach): Add test for deterministic cuDNN max-pooling
LayerShapeNHWC = collections.namedtuple('LayerShapeNHWC',
'batch, height, width, channels')
FilterShape2D = collections.namedtuple(
'FilterShape2D', 'height, width, in_channels, out_channels')
FilterShape2DTranspose = collections.namedtuple(
'FilterShape2DTranspose', 'height, width, out_channels, in_channels')
LayerShapeNCDHW = collections.namedtuple(
'LayerShapeNCDHW', 'batch, channels, depth, height, width')
FilterShape3D = collections.namedtuple(
'FilterShape3D', 'depth, height, width, in_channels, out_channels')
class ConvolutionTest(test.TestCase):
"""Tests for deterministic cuDNN functionality."""
def _random_data_op(self, shape):
# np.random.random_sample can properly interpret either tf.TensorShape or
# namedtuple as a list.
return constant_op.constant(
2 * np.random.random_sample(shape) - 1, dtype=dtypes.float32)
def _random_out_op(self, in_shape, filter_shape, strides, padding, dilations):
# Choosing not to use array_op.zeros() to prevent possible removal by
# optimization
in_op = self._random_data_op(in_shape)
filter_op = self._random_data_op(filter_shape)
# Use the forward op's shape-inference
conv_op = nn_ops.conv2d(
in_op, filter_op, strides=strides, padding=padding, dilations=dilations)
out_shape = conv_op.get_shape()
out_op = self._random_data_op(out_shape)
return out_op
def _assert_reproducible(self, operation):
with test_util.force_gpu():
result_1 = operation()
result_2 = operation()
self.assertAllEqual(result_1, result_2)
# The default forward algorithm choice, when using cuDNN 7, does not support
# the following layer configuration. This test case intends to confirm that
# an alternative algorithm is selected. Note that, in cuDNN 7, all forward
# algorithms are determnistic.
@test_util.run_cuda_only
def testConvForwardDefaultAlgorithmChoice(self):
in_shape = LayerShapeNCDHW(batch=2, channels=3, depth=5, height=7, width=6)
filter_shape = FilterShape3D(
depth=3, height=3, width=3, in_channels=3, out_channels=2)
in_op = self._random_data_op(in_shape)
filter_op = self._random_data_op(filter_shape)
self._assert_reproducible(lambda: nn_ops.conv3d(
in_op,
filter_op,
strides=[1, 1, 1, 1, 1],
padding='VALID',
data_format='NCDHW',
dilations=[1, 1, 2, 2, 2]))
# This test is primarily testing XLA since cuDNN forward convolutions are
# always deterministic, even when determinism is not enabled. The convolution
# configuration tested is nondeterministic with XLA when determinism is not
# enabled.
@test_util.run_cuda_only
def testConvForwardXLA(self):
in_shape = LayerShapeNCDHW(
batch=2, channels=8, depth=5, height=12, width=15)
filter_shape = FilterShape3D(
depth=3, height=3, width=3, in_channels=8, out_channels=1)
in_op = self._random_data_op(in_shape)
filter_op = self._random_data_op(filter_shape)
self._assert_reproducible(lambda: nn_ops.conv3d(
in_op,
filter_op,
strides=[1, 1, 1, 1, 1],
padding='VALID',
data_format='NCDHW',
dilations=[1, 1, 2, 2, 2]))
@test_util.run_cuda_only
def testConvBackwardFilterGradient(self, rate=1):
in_shape = LayerShapeNHWC(batch=8, height=64, width=64, channels=8)
filter_shape = FilterShape2D(
height=3, width=3, in_channels=8, out_channels=8)
in_op = self._random_data_op(in_shape)
strides = [1, 1, 1, 1]
padding = 'SAME'
dilations = [1, rate, rate, 1]
out_op = self._random_out_op(in_shape, filter_shape, strides, padding,
dilations)
self._assert_reproducible(lambda: nn_ops.conv2d_backprop_filter(
in_op,
filter_shape,
out_op,
strides=strides,
padding=padding,
dilations=dilations))
# A configuration for this test could not be found that exercises
# nondeterminism when using XLA with determinism not enabled.
@test_util.run_cuda_only
def testConvBackwardFilterGradientWithDilations(self):
self.testConvBackwardFilterGradient(rate=2)
@test_util.run_cuda_only
def testConvBackwardInputGradient(self, rate=1):
in_shape = LayerShapeNHWC(batch=1, height=16, width=16, channels=1)
filter_shape = FilterShape2D(
height=7, width=7, in_channels=1, out_channels=3)
filter_op = self._random_data_op(filter_shape)
strides = [1, 1, 1, 1]
padding = 'SAME'
dilations = [1, rate, rate, 1]
out_op = self._random_out_op(in_shape, filter_shape, strides, padding,
dilations)
self._assert_reproducible(lambda: nn_ops.conv2d_backprop_input(
in_shape,
filter_op,
out_op,
strides=strides,
padding=padding,
dilations=dilations))
# A configuration for this test could not be found that exercises
# nondeterminism when using XLA with determinism not enabled.
@test_util.run_cuda_only
def testConvBackwardInputGradientWithDilations(self):
self.testConvBackwardInputGradient(rate=2)
@test_util.run_cuda_only
def testConvTransposeForward(self, rate=1):
in_channels = 3
out_channels = 1
in_shape = LayerShapeNHWC(
batch=1, height=16, width=16, channels=in_channels)
filter_shape = FilterShape2DTranspose(
height=7, width=7, out_channels=out_channels, in_channels=in_channels)
in_op = self._random_data_op(in_shape)
filter_op = self._random_data_op(filter_shape)
out_shape = LayerShapeNHWC(
batch=in_shape.batch,
height=in_shape.height,
width=in_shape.width,
channels=out_channels)
self._assert_reproducible(lambda: nn_ops.conv2d_transpose_v2(
in_op,
filter_op,
out_shape,
strides=1,
padding='SAME',
data_format='NHWC',
dilations=[1, rate, rate, 1]))
# A configuration for this test could not be found that exercises
# nondeterminism when using XLA with determinism not enabled.
@test_util.run_cuda_only
def testConvTransposeForwardWithDilations(self):
self.testConvTransposeForward(rate=2)
@test_util.run_cuda_only
def testConvTransposeBackwardFilterGradient(self, rate=1):
in_channels = 8
out_channels = 8
in_shape = LayerShapeNHWC(
batch=8, height=64, width=64, channels=in_channels)
filter_shape = FilterShape2DTranspose(
height=3, width=3, out_channels=out_channels, in_channels=in_channels)
in_op = self._random_data_op(in_shape)
filter_op = self._random_data_op(filter_shape)
out_shape = LayerShapeNHWC(
batch=in_shape.batch,
height=in_shape.height,
width=in_shape.width,
channels=out_channels)
upstream_gradients = self._random_data_op(out_shape)
def gradient():
with backprop.GradientTape() as tape:
tape.watch(filter_op)
op_output = nn_ops.conv2d_transpose_v2(
in_op,
filter_op,
out_shape,
strides=1,
padding='SAME',
data_format='NHWC',
dilations=[1, rate, rate, 1])
gradient_injector_output = op_output * upstream_gradients
return tape.gradient(gradient_injector_output, [filter_op])[0]
self._assert_reproducible(gradient)
# A configuration for this test could not be found that exercises
# nondeterminism when using XLA with determinism not enabled.
@test_util.run_cuda_only
def testConvTransposeBackwardFilterGradientWithDilations(self):
self.testConvTransposeBackwardFilterGradient(rate=2)
# A configuration for this test could not be found that exercises
# nondeterminism when determinism is not enabled (for either XLA or non-XLA).
@test_util.run_cuda_only
def testConvTransposeBackwardInputGradient(self, rate=1):
in_channels = 1
out_channels = 3
in_shape = LayerShapeNHWC(
batch=1, height=16, width=16, channels=in_channels)
filter_shape = FilterShape2DTranspose(
height=7, width=7, out_channels=out_channels, in_channels=in_channels)
in_op = self._random_data_op(in_shape)
filter_op = self._random_data_op(filter_shape)
out_shape = LayerShapeNHWC(
batch=in_shape.batch,
height=in_shape.height,
width=in_shape.width,
channels=out_channels)
upstream_gradients = self._random_data_op(out_shape)
def gradient():
with backprop.GradientTape() as tape:
tape.watch(in_op)
op_output = nn_ops.conv2d_transpose_v2(
in_op,
filter_op,
out_shape,
strides=1,
padding='SAME',
data_format='NHWC',
dilations=[1, rate, rate, 1])
gradient_injector_output = op_output * upstream_gradients
return tape.gradient(gradient_injector_output, [in_op])[0]
self._assert_reproducible(gradient)
# A configuration for this test could not be found that exercises
# nondeterminism when determinism is not enabled (for either XLA or non-XLA).
@test_util.run_cuda_only
def testConvTransposeBackwardInputGradientWithDilations(self):
self.testConvTransposeBackwardInputGradient(rate=2)
tensorflow/python/kernel_tests/nn_ops/cudnn_deterministic_ops_test.py
+29
View File
@@ -0,0 +1,29 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for TF_DETERMINISTIC_OPS=1."""
import os
from tensorflow.python.framework import config
from tensorflow.python.kernel_tests.nn_ops import cudnn_deterministic_base
from tensorflow.python.platform import test
ConvolutionTest = cudnn_deterministic_base.ConvolutionTest
if __name__ == '__main__':
# TODO(reedwm): Merge this file with cudnn_deterministic_base.py.
config.enable_op_determinism()
os.environ['TF_DETERMINISTIC_OPS'] = '1'
test.main()
tensorflow/python/kernel_tests/nn_ops/depthwise_conv_op_base.py
+1172
View File
File diff suppressed because it is too large Load Diff
tensorflow/python/kernel_tests/nn_ops/depthwise_conv_op_d9m_test.py
+170
View File
@@ -0,0 +1,170 @@
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Functional tests for determinsitic depthwise convolutional operations."""
from tensorflow.python.eager import backprop
from tensorflow.python.framework import config as tf_config
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import random_seed
from tensorflow.python.framework import test_util
from tensorflow.python.kernel_tests.nn_ops import depthwise_conv_op_base
from tensorflow.python.ops import nn_impl
from tensorflow.python.ops import random_ops
# The following imports are required to register the gradient functions.
from tensorflow.python.ops.nn_grad import _DepthwiseConv2dNativeBackpropFilterGrad # pylint: disable=unused-import
from tensorflow.python.ops.nn_grad import _DepthwiseConv2dNativeBackpropInputGrad # pylint: disable=unused-import
from tensorflow.python.platform import test
@test_util.run_all_without_tensor_float_32("Uses matmul")
class DepthwiseConv2DDeterministicTest(
depthwise_conv_op_base.DepthwiseConv2DBase):
"""Test determinism-related functionality of tf.nn.depthwise_conv2d."""
def _genParams(self,
use_cudnn=False,
data_format="NHWC",
dtype=dtypes.float32,
seed=123):
random_seed.set_seed(seed)
batch_size = 2 # no interaction over batch, so make small
if use_cudnn:
# When op-determinism is not enabled, one input channel, plus a
# cuDNN-supported filter size and number of output channels will result
# in cuDNN being used for both backprop-to-input and backprop-to-filter on
# cuDNN 7.6.3 and higher. When op-determnism is enabled, cuDNN is always
# used for backprop-to-filter.
input_channels = 1
else:
input_channels = 2 # no interaction over channels, so make small
input_height = 500
input_width = 1000
if data_format == "NHWC":
input_shape = (batch_size, input_height, input_width, input_channels)
else: # "NCHW"
input_shape = (batch_size, input_channels, input_height, input_width)
input_data = random_ops.random_normal(input_shape, dtype=dtype)
# The following filter size results in nondeterminism being exercised in
# cuDNN backprop (when determinism is not enabled) to both input and filter
# as well as in the specialized (non-cuDNN) depthwise backprop to filter.
filter_height = 7
filter_width = 7
channel_multiplier = 10
filter_shape = (filter_height, filter_width, input_channels,
channel_multiplier)
filter_data = random_ops.random_normal(filter_shape, dtype=dtype)
strides = [1, 1, 1, 1]
padding = "SAME"
output_height = input_height # because same padding
output_width = input_width # because same padding
output_channels = input_channels * channel_multiplier
if data_format == "NHWC":
output_shape = (batch_size, output_height, output_width, output_channels)
else: # "NCHW"
output_shape = (batch_size, output_channels, output_height, output_width)
return input_data, filter_data, strides, padding, output_shape
def _testForwardDeterminismCase(self,
use_cudnn=False,
data_format="NHWC",
dtype=dtypes.float32):
for seed in range(5):
p = self._genParams(use_cudnn, data_format, dtype, seed=seed)
input_data, filter_data, strides, padding, _ = p
result_a = nn_impl.depthwise_conv2d_v2(input_data, filter_data, strides,
padding, data_format)
result_b = nn_impl.depthwise_conv2d_v2(input_data, filter_data, strides,
padding, data_format)
self.assertAllEqual(result_a, result_b)
@test_util.run_gpu_only
def testForwardDeterminismGPU(self):
for use_cudnn in [False, True]:
for data_format in ["NHWC", "NCHW"]:
for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]:
self._testForwardDeterminismCase(use_cudnn, data_format, dtype=dtype)
def testForwardDeterminismCPU(self):
if tf_config.list_physical_devices("GPU"):
self.skipTest("Test only runs when there is no GPU")
data_format = "NHWC" # CPU does not implement NCHW version of op
for dtype in [dtypes.bfloat16.as_numpy_dtype, dtypes.float32,
dtypes.float64]:
self._testForwardDeterminismCase(data_format=data_format, dtype=dtype)
def _testBackwardDeterminismCase(self,
using_gpu=False,
use_cudnn=False,
data_format="NHWC",
dtype=dtypes.float32):
p = self._genParams(use_cudnn, data_format, dtype, seed=123)
input_data, filter_data, strides, padding, output_shape = p
def Gradients(upstream_gradients):
with backprop.GradientTape() as tape:
tape.watch(input_data)
tape.watch(filter_data)
op_output = nn_impl.depthwise_conv2d_v2(input_data, filter_data,
strides, padding, data_format)
gradient_injector_output = op_output * upstream_gradients
return tape.gradient(gradient_injector_output, [input_data, filter_data])
# Test only two seeds, since testing takes a long time
for seed in (987, 988):
upstream_gradients = random_ops.random_normal(
output_shape, dtype=dtype, seed=seed)
input_gradients_a, filter_gradients_a = Gradients(upstream_gradients)
input_gradients_b, filter_gradients_b = Gradients(upstream_gradients)
self.assertAllEqual(input_gradients_a, input_gradients_b)
self.assertAllEqual(filter_gradients_a, filter_gradients_b)
@test_util.run_gpu_only
def testBackwardDeterminismGPU(self):
using_gpu = True
for use_cudnn in [False, True]:
for data_format in ["NHWC", "NCHW"]:
for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]:
self._testBackwardDeterminismCase(using_gpu, use_cudnn, data_format,
dtype)
def testBackwardDeterminismCPU(self):
if tf_config.list_physical_devices("GPU"):
self.skipTest("Test only runs when there is no GPU")
data_format = "NHWC" # CPU does not implement NCHW version of op
for dtype in [dtypes.bfloat16.as_numpy_dtype, dtypes.float32,
dtypes.float64]:
self._testBackwardDeterminismCase(data_format=data_format, dtype=dtype)
if __name__ == "__main__":
# The op-determinism setting can be enabled and disabled on-the-fly.
# However, if cuDNN convolution is used (as it is for these tests) then its
# setting at the time will influence which algorithm for a particular layer
# configuration is cached (independently for XLA and non-XLA operation).
#
# The tests in this file must be run under a separate test.main from the
# tests in depthwise_conv_op_test.py to prevent caching the selection of
# nondeterminsitic algorithms, which would cause the tests defined in this
# file to fail.
#
# Also because of this caching, the tests defined in depthwise_conv_op_base.py
# should be run with and without op-determinism enabled in separate files.
#
# TODO(duncanriach): Implement cuDNN auto-tuning cache invalidation and
# and execute when op-determinism setting is changed.
tf_config.enable_op_determinism()
test.main()
tensorflow/python/kernel_tests/nn_ops/depthwise_conv_op_test.py
+23
View File
@@ -0,0 +1,23 @@
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Functional tests for depthwise convolutional operations."""
from tensorflow.python.kernel_tests.nn_ops import depthwise_conv_op_base
from tensorflow.python.platform import test
DepthwiseConv2DTest = depthwise_conv_op_base.DepthwiseConv2DBase
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/embedding_ops_test.py
+1443
View File
File diff suppressed because it is too large Load Diff
tensorflow/python/kernel_tests/nn_ops/fractional_avg_pool_op_test.py
+612
View File
@@ -0,0 +1,612 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for fractional average pool operation."""
import math
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_nn_ops
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
class FractionalAvgTest(test.TestCase):
# Random number generate with seed.
_PRNG = np.random.RandomState(341261000)
_SEED = 341261001
def _AvgPoolAlongRows(self, input_matrix, row_seq, overlapping):
"""Perform average pool along row of a 2-D matrix based on row_seq.
Args:
input_matrix: A 2-D matrix.
row_seq: Cumulative pooling sequence along row.
overlapping: Whether or not use overlapping when pooling.
Returns:
A 2-D matrix, with
* num_rows = len(row_seq)-1
* num_cols = input_matrix.num_cols.
"""
output_image = np.zeros(input_matrix.shape[1])
row_max = row_seq[-1]
for i in range(row_seq.shape[0] - 1):
row_start = row_seq[i]
row_end = row_seq[i + 1] + 1 if overlapping else row_seq[i + 1]
row_end = min(row_end, row_max)
output_image = np.vstack((output_image, np.mean(
input_matrix[row_start:row_end, :], axis=0))) # axis 0 is along row
# remove the sentinel row
return output_image[1:, :]
def _AvgPoolAlongCols(self, input_matrix, col_seq, overlapping):
"""Perform average pool along column of a 2-D matrix based on col_seq.
Args:
input_matrix: A 2-D matrix.
col_seq: Cumulative pooling sequence along column.
overlapping: Whether or not use overlapping when pooling.
Returns:
A 2-D matrix, with
* num_rows = input_matrix.num_rows
* num_cols = len(col_seq)-1.
"""
input_matrix = input_matrix.transpose()
output_matrix = self._AvgPoolAlongRows(input_matrix, col_seq, overlapping)
return output_matrix.transpose()
def _GetExpectedFractionalAvgPoolResult(self, input_tensor, row_seq, col_seq,
overlapping):
"""Get expected fractional average pooling result.
row_seq and col_seq together defines the fractional pooling region.
Args:
input_tensor: Original input tensor, assuming it is a 4-D tensor, with
dimension as [batch, height/row, width/column, channels/depth].
row_seq: Cumulative pooling sequence along row.
col_seq: Cumulative pooling sequence along column.
overlapping: Use overlapping when doing pooling.
Returns:
A 4-D tensor that is the result of average pooling on input_tensor based
on pooling region defined by row_seq and col_seq, conditioned on whether
or not overlapping is used.
"""
input_shape = input_tensor.shape
output_shape = (input_shape[0], len(row_seq) - 1, len(col_seq) - 1,
input_shape[3])
output_tensor = np.zeros(shape=output_shape, dtype=input_tensor.dtype)
for batch in range(input_shape[0]):
for channel in range(input_shape[3]):
two_dim_slice = input_tensor[batch, :, :, channel]
tmp = self._AvgPoolAlongRows(two_dim_slice, row_seq, overlapping)
output_tensor[batch, :, :, channel] = self._AvgPoolAlongCols(
tmp, col_seq, overlapping)
return output_tensor
def _ValidateFractionalAvgPoolResult(self, input_tensor, pooling_ratio,
pseudo_random, overlapping):
"""Validate FractionalAvgPool's result against expected.
Expected result is computed given input_tensor, and pooling region defined
by row_seq and col_seq.
Args:
input_tensor: A tensor or numpy ndarray.
pooling_ratio: A list or tuple of length 4, first and last element be 1.
pseudo_random: Use pseudo random method to generate pooling sequence.
overlapping: Use overlapping when pooling.
Returns:
None
"""
with self.cached_session() as sess:
p, r, c = nn_ops.fractional_avg_pool_v2(
input_tensor,
pooling_ratio,
pseudo_random,
overlapping,
seed=self._SEED)
actual, row_seq, col_seq = self.evaluate([p, r, c])
expected = self._GetExpectedFractionalAvgPoolResult(input_tensor, row_seq,
col_seq, overlapping)
self.assertShapeEqual(expected, p)
self.assertAllClose(expected, actual)
def _testVisually(self):
"""Manual test by printing out intermediate result of a small random tensor.
Since _GetExpectedFractionalAvgPoolResult is 'automated', it feels safer to
have a test case that you can see what's happening.
This test will generate a small, random, int 2D matrix, and feed it to
FractionalAvgPool and _GetExpectedFractionalAvgPoolResult.
"""
num_rows = 6
num_cols = 6
tensor_shape = (1, num_rows, num_cols, 1)
pseudo_random = False
for overlapping in True, False:
print("-" * 70)
print("Testing FractionalAvgPool with overlapping = {}".format(
overlapping))
rand_mat = self._PRNG.randint(10, size=tensor_shape)
pooling_ratio = [1, math.sqrt(2), math.sqrt(2), 1]
with self.cached_session() as sess:
p, r, c = nn_ops.fractional_avg_pool_v2(
rand_mat.astype(np.float32),
pooling_ratio,
pseudo_random,
overlapping,
seed=self._SEED)
tensor_output, row_seq, col_seq = self.evaluate([p, r, c])
expected_result = self._GetExpectedFractionalAvgPoolResult(
rand_mat.astype(np.float32), row_seq, col_seq, overlapping)
print("row sequence:")
print(row_seq)
print("column sequence:")
print(col_seq)
print("Input:")
# Print input with pooling region marked.
for i in range(num_rows):
row_to_print = []
for j in range(num_cols):
if j in col_seq:
row_to_print.append("|")
row_to_print.append(str(rand_mat[0, i, j, 0]))
row_to_print.append("|")
if i in row_seq:
print("-" * 2 * len(row_to_print))
print(" ".join(row_to_print))
print("-" * 2 * len(row_to_print))
print("Output from FractionalAvgPool:")
print(tensor_output[0, :, :, 0])
print("Expected result:")
print(expected_result[0, :, :, 0])
def testAllInputOptions(self):
"""Try all possible input options for fractional_avg_pool.
"""
num_batches = 5
num_channels = 3
num_rows = 20
num_cols = 30
for pseudo_random in True, False:
for overlapping in True, False:
tensor_shape = (num_batches, num_rows, num_cols, num_channels)
# random tensor with value in [-500.0, 500.0)
rand_mat = self._PRNG.random_sample(tensor_shape) * 1000 - 500
self._ValidateFractionalAvgPoolResult(
rand_mat, [1, math.sqrt(3), math.sqrt(2), 1], pseudo_random,
overlapping)
def testIntegerTensorInput(self):
"""Test FractionalAvgPool works fine when input tensor is integer type.
"""
pseudo_random = True
overlapping = True
tensor_shape = (1, 6, 6, 1)
# pyformat: disable
mat = np.array([
[2, 6, 4, 1, 3, 6],
[8, 9, 1, 6, 6, 8],
[3, 9, 8, 2, 5, 6],
[2, 7, 9, 5, 4, 5],
[8, 5, 0, 5, 7, 4],
[4, 4, 5, 9, 7, 2]
])
# pyformat: enable
self._ValidateFractionalAvgPoolResult(mat.reshape(tensor_shape),
[1, math.sqrt(3), math.sqrt(2), 1],
pseudo_random, overlapping)
def testDifferentTensorShapes(self):
"""Test different shapes of input tensor.
Mainly test different combinations of num_rows and num_cols.
"""
pseudo_random = True
overlapping = True
for num_batches in [1, 3]:
for num_channels in [1, 3]:
for num_rows in [10, 20, 50]:
for num_cols in [10, 20, 50]:
tensor_shape = (num_batches, num_rows, num_cols, num_channels)
# random tensor with value in [-500.0, 500.0)
rand_mat = self._PRNG.random_sample(tensor_shape) * 1000 - 500
self._ValidateFractionalAvgPoolResult(
rand_mat, [1, math.sqrt(3), math.sqrt(2), 1], pseudo_random,
overlapping)
def testLargePoolingRatio(self):
"""Test when pooling ratio is not within [1, 2).
"""
pseudo_random = True
overlapping = True
num_batches = 3
num_channels = 3
num_rows = 30
num_cols = 50
tensor_shape = (num_batches, num_rows, num_cols, num_channels)
for row_ratio in [math.sqrt(11), math.sqrt(37)]:
for col_ratio in [math.sqrt(11), math.sqrt(27)]:
# random tensor with value in [-500.0, 500.0)
rand_mat = self._PRNG.random_sample(tensor_shape) * 1000 - 500
self._ValidateFractionalAvgPoolResult(rand_mat,
[1, row_ratio, col_ratio, 1],
pseudo_random, overlapping)
def testDivisiblePoolingRatio(self):
"""Test when num of rows/cols can evenly divide pooling ratio.
This is a case regular average pooling can handle. Should be handled by
fractional pooling as well.
"""
pseudo_random = True
overlapping = True
num_batches = 3
num_channels = 3
num_rows = 30
num_cols = 50
tensor_shape = (num_batches, num_rows, num_cols, num_channels)
# random tensor with value in [-500.0, 500.0)
rand_mat = self._PRNG.random_sample(tensor_shape) * 1000 - 500
self._ValidateFractionalAvgPoolResult(rand_mat, [1, 2, 2, 1], pseudo_random,
overlapping)
@test_util.run_deprecated_v1
def testDifferentInputTensorShape(self):
"""Runs the operation in one session with different input tensor shapes."""
with self.cached_session() as sess:
input_holder = array_ops.placeholder(dtypes.float32,
[None, None, None, 3])
pooling_ratio = [1, 1.5, 1.5, 1]
pseudo_random = False
overlapping = False
p, r, c = nn_ops.fractional_avg_pool_v2(
input_holder,
pooling_ratio,
pseudo_random,
overlapping,
seed=self._SEED)
# First run.
input_a = np.zeros([3, 32, 32, 3])
actual, row_seq, col_seq = sess.run([p, r, c], {input_holder: input_a})
expected = self._GetExpectedFractionalAvgPoolResult(
input_a, row_seq, col_seq, overlapping)
self.assertSequenceEqual(expected.shape, actual.shape)
# Second run.
input_b = np.zeros([4, 60, 60, 3])
actual, row_seq, col_seq = sess.run([p, r, c], {input_holder: input_b})
expected = self._GetExpectedFractionalAvgPoolResult(
input_b, row_seq, col_seq, overlapping)
self.assertSequenceEqual(expected.shape, actual.shape)
def testNegativeSeqValuesForGradOp(self):
with self.assertRaisesRegex(
errors.InvalidArgumentError,
r"Row sequence tensor values must not be negative.*"):
y = nn_ops.gen_nn_ops.fractional_avg_pool_grad(
orig_input_tensor_shape=[2, 2, 2, 2],
out_backprop=[[[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [11,
12]]]],
row_pooling_sequence=[-10, 1, 2, 3],
col_pooling_sequence=[1, 2, 3, 4],
overlapping=True)
self.evaluate(y)
with self.assertRaisesRegex(
errors.InvalidArgumentError,
r"Column sequence tensor values must not be negative.*"):
z = nn_ops.gen_nn_ops.fractional_avg_pool_grad(
orig_input_tensor_shape=[2, 2, 2, 2],
out_backprop=[[[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [11,
12]]]],
row_pooling_sequence=[10, 1, 2, 3],
col_pooling_sequence=[1, 2, -3, 4],
overlapping=True)
self.evaluate(z)
def testPoolingRatioHasMoreDimThanInput(self):
with self.cached_session() as _:
with self.assertRaisesRegex(
errors.InvalidArgumentError,
r"Pooling ratio is higher than input dimension size for dimension 1.*"
):
result = nn_ops.gen_nn_ops.fractional_avg_pool(
value=constant_op.constant(
value=[[[[1, 4, 2, 3]]]], dtype=dtypes.int64),
pooling_ratio=[1.0, 1.44, 1.73, 1.0],
pseudo_random=False,
overlapping=False,
deterministic=False,
seed=0,
seed2=0,
name=None)
self.evaluate(result)
def testPoolingRatioIllegalSmallValue(self):
with self.cached_session() as _:
# Whether turn on `TF2_BEHAVIOR` generates different error messages
with self.assertRaisesRegex(
(errors.InvalidArgumentError, ValueError),
r"(pooling_ratio cannot be smaller than 1, got: .*)|(is negative)"):
result = nn_ops.gen_nn_ops.fractional_avg_pool(
value=np.zeros([3, 30, 30, 3]),
pooling_ratio=[1, -1, 3, 1],
pseudo_random=False,
overlapping=False,
deterministic=False,
seed=0,
seed2=0,
)
self.evaluate(result)
def testPoolingIllegalRatioForBatch(self):
with self.cached_session() as _:
with self.assertRaises(errors.UnimplementedError):
result = nn_ops.gen_nn_ops.fractional_avg_pool(
np.zeros([3, 30, 50, 3]),
[2, 3, 1.5, 1],
True,
True)
self.evaluate(result)
class FractionalAvgPoolGradTest(test.TestCase):
"""Tests for FractionalAvgPoolGrad.
Two types of tests for FractionalAvgPoolGrad.
1) Test fractional_avg_pool_grad() directly.
This type of test relies on gen_nn_ops.avg_pool_grad() returns the
correct result. For example:
* input_tensor_shape = (1, 10, 10, 1)
* window_size = (1, 2, 2, 1)
* stride_size = (1, 2, 2, 1)
* padding: not really important, since 10/2 is divisible
avg pooling should generate the same result as fractional avg pooling with:
* row_sequence = [0, 2, 4, 6, 8, 10]
* col_sequence = [0, 2, 4, 6, 8, 10]
* overlapping = False
This also means their gradients in such case will be the same.
Similarly, when
* input_tensor_shape = (1, 7, 7, 1)
* window_size = (1, 3, 3, 1)
* stride_size = (1, 2, 2, 1)
* padding: not important
avg pooling should generate the same result as fractional avg pooling with:
* row_sequence = [0, 2, 4, 7]
* col_sequence = [0, 2, 4, 7]
* overlapping = True
2) Test through compute_gradient_error()
"""
_PRNG = np.random.RandomState(341261004)
_SEED = 341261005
def _GenerateRandomInputTensor(self, shape):
num_elements = 1
for dim_size in shape:
num_elements *= dim_size
x = self._PRNG.rand(num_elements) * 1000
return x.reshape(shape)
def testDirectNotUseOverlapping(self):
for num_batches in [1, 3]:
for row_window_size in [2, 5]:
for col_window_size in [2, 4]:
num_rows = row_window_size * 5
num_cols = col_window_size * 7
for num_channels in [1, 2]:
input_shape = (num_batches, num_rows, num_cols, num_channels)
with self.cached_session() as _:
input_tensor = constant_op.constant(
self._GenerateRandomInputTensor(input_shape).astype(
np.float32))
window_size = [1, row_window_size, col_window_size, 1]
stride_size = [1, row_window_size, col_window_size, 1]
padding = "VALID"
output_tensor = nn_ops.avg_pool(input_tensor, window_size,
stride_size, padding)
output_data = self.evaluate(output_tensor)
num_elements = 1
for dim_size in output_data.shape:
num_elements *= dim_size
output_backprop = (self._PRNG.rand(num_elements) *
1000).reshape(output_data.shape)
input_backprop_tensor = gen_nn_ops.avg_pool_grad(
input_tensor.get_shape(), output_backprop, window_size,
stride_size, padding)
input_backprop = self.evaluate(input_backprop_tensor)
row_seq = list(range(0, num_rows + 1, row_window_size))
col_seq = list(range(0, num_cols + 1, col_window_size))
fap_input_backprop_tensor = gen_nn_ops.fractional_avg_pool_grad(
input_tensor.get_shape(),
output_backprop,
row_seq,
col_seq,
overlapping=False)
fap_input_backprop = self.evaluate(fap_input_backprop_tensor)
self.assertShapeEqual(input_backprop, fap_input_backprop_tensor)
self.assertAllClose(input_backprop, fap_input_backprop)
def testDirectUseOverlapping(self):
for num_batches in [1, 3]:
for row_window_size in [2, 5]:
for col_window_size in [2, 4]:
num_rows = (row_window_size - 1) * 5 + 1
num_cols = (col_window_size - 1) * 7 + 1
for num_channels in [1, 2]:
input_shape = (num_batches, num_rows, num_cols, num_channels)
with self.cached_session() as _:
input_tensor = constant_op.constant(
self._GenerateRandomInputTensor(input_shape).astype(
np.float32))
window_size = [1, row_window_size, col_window_size, 1]
stride_size = [1, row_window_size - 1, col_window_size - 1, 1]
padding = "VALID"
output_tensor = nn_ops.avg_pool(input_tensor, window_size,
stride_size, padding)
output_data = self.evaluate(output_tensor)
num_elements = 1
for dim_size in output_data.shape:
num_elements *= dim_size
output_backprop = (self._PRNG.rand(num_elements) *
1000).reshape(output_data.shape)
input_backprop_tensor = gen_nn_ops.avg_pool_grad(
input_tensor.get_shape(), output_backprop, window_size,
stride_size, padding)
input_backprop = self.evaluate(input_backprop_tensor)
row_seq = list(range(0, num_rows, row_window_size - 1))
col_seq = list(range(0, num_cols, col_window_size - 1))
row_seq[-1] += 1
col_seq[-1] += 1
fap_input_backprop_tensor = gen_nn_ops.fractional_avg_pool_grad(
input_tensor.get_shape(),
output_backprop,
row_seq,
col_seq,
overlapping=True)
fap_input_backprop = self.evaluate(fap_input_backprop_tensor)
self.assertShapeEqual(input_backprop, fap_input_backprop_tensor)
self.assertAllClose(input_backprop, fap_input_backprop)
@test_util.run_deprecated_v1
def testAllInputOptionsThroughGradientError(self):
input_shape = (1, 7, 13, 1)
input_data = self._GenerateRandomInputTensor(input_shape)
pooling_ratio = [1, math.sqrt(2), math.sqrt(3), 1]
for pseudo_random in True, False:
for overlapping in True, False:
with self.cached_session() as _:
input_tensor = constant_op.constant(input_data, shape=input_shape)
output_tensor, unused_a, unused_b = nn_ops.fractional_avg_pool_v2(
input_tensor,
pooling_ratio,
pseudo_random=pseudo_random,
overlapping=overlapping,
seed=self._SEED)
output_data = self.evaluate(output_tensor)
output_shape = output_data.shape
# error_margin and delta setting is similar to avg_pool_grad.
error_margin = 1e-4
gradient_error = gradient_checker.compute_gradient_error(
input_tensor,
input_shape,
output_tensor,
output_shape,
x_init_value=input_data.reshape(input_shape),
delta=1e-2)
self.assertLess(gradient_error, error_margin)
@test_util.run_deprecated_v1
def testDifferentTensorShapesThroughGradientError(self):
pseudo_random = True
overlapping = True
pooling_ratio = [1, math.sqrt(3), math.sqrt(2), 1]
for num_batches in [1, 2]:
for num_rows in [5, 13]:
for num_cols in [5, 11]:
for num_channels in [1, 3]:
input_shape = (num_batches, num_rows, num_cols, num_channels)
input_data = self._GenerateRandomInputTensor(input_shape)
with self.cached_session() as _:
input_tensor = constant_op.constant(input_data, shape=input_shape)
output_tensor, unused_a, unused_b = nn_ops.fractional_avg_pool_v2(
input_tensor,
pooling_ratio,
pseudo_random=pseudo_random,
overlapping=overlapping,
seed=self._SEED)
output_data = self.evaluate(output_tensor)
output_shape = output_data.shape
# error_margin and delta setting is similar to avg_pool_grad.
error_margin = 1e-4
gradient_error = gradient_checker.compute_gradient_error(
input_tensor,
input_shape,
output_tensor,
output_shape,
x_init_value=input_data.reshape(input_shape),
delta=1e-2)
self.assertLess(gradient_error, error_margin)
@test_util.run_deprecated_v1
def testLargePoolingRatioThroughGradientError(self):
input_shape = (1, 17, 23, 1)
input_data = self._GenerateRandomInputTensor(input_shape)
pooling_ratio = (1, math.sqrt(13), math.sqrt(7), 1)
output_shape = [int(a / b) for a, b in zip(input_shape, pooling_ratio)]
overlapping = True
pseudo_random = False
with self.cached_session() as _:
input_tensor = constant_op.constant(input_data, shape=input_shape)
output_tensor, unused_a, unused_b = nn_ops.fractional_avg_pool_v2(
input_tensor,
pooling_ratio,
pseudo_random=pseudo_random,
overlapping=overlapping,
seed=self._SEED)
# error_margin and delta setting is similar to avg_pool_grad.
error_margin = 1e-4
gradient_error = gradient_checker.compute_gradient_error(
input_tensor,
input_shape,
output_tensor,
output_shape,
x_init_value=input_data.reshape(input_shape),
delta=1e-2)
self.assertLess(gradient_error, error_margin)
def testInvalidSeqRaiseErrorForFractionalAvgPoolGrad(self):
with self.assertRaises((errors.InvalidArgumentError, ValueError)):
with self.cached_session() as _:
overlapping = True
orig_input_tensor_shape = constant_op.constant(
-1879048192, shape=[4], dtype=dtypes.int64)
out_backprop = constant_op.constant([],
shape=[0, 0, 0, 0],
dtype=dtypes.float64)
row_pooling_sequence = constant_op.constant(
1, shape=[4], dtype=dtypes.int64)
col_pooling_sequence = constant_op.constant(
1, shape=[4], dtype=dtypes.int64)
t = gen_nn_ops.fractional_avg_pool_grad(
orig_input_tensor_shape=orig_input_tensor_shape,
out_backprop=out_backprop,
row_pooling_sequence=row_pooling_sequence,
col_pooling_sequence=col_pooling_sequence,
overlapping=overlapping)
self.evaluate(t)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/fractional_max_pool_op_test.py
+733
View File
@@ -0,0 +1,733 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for fractional max pool operation."""
import math
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_nn_ops
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
class FractionalMaxPoolTest(test.TestCase):
# Random number generate with seed.
_PRNG = np.random.RandomState(341261)
_SEED = 123456
def _MaxPoolAlongRows(self, input_matrix, row_seq, overlapping):
"""Perform max pool along row of a 2-D matrix based on row_seq.
Args:
input_matrix: A 2-D matrix.
row_seq: Cumulative pooling sequence along row.
overlapping: Whether or not use overlapping when pooling.
Returns:
A 2-D matrix, with
* num_rows = len(row_seq)-1
* num_cols = input_matrix.num_cols.
"""
output_image = np.zeros(input_matrix.shape[1])
row_max = row_seq[-1]
for i in range(row_seq.shape[0] - 1):
row_start = row_seq[i]
row_end = row_seq[i + 1] + 1 if overlapping else row_seq[i + 1]
row_end = min(row_end, row_max)
output_image = np.vstack((output_image, np.amax(
input_matrix[row_start:row_end, :], axis=0))) # axis 0 is along row
# remove the sentinel row
return output_image[1:, :]
def _MaxPoolAlongCols(self, input_matrix, col_seq, overlapping):
"""Perform max pool along column of a 2-D matrix based on col_seq.
Args:
input_matrix: A 2-D matrix.
col_seq: Cumulative pooling sequence along column.
overlapping: Whether or not use overlapping when pooling.
Returns:
A 2-D matrix, with
* num_rows = input_matrix.num_rows
* num_cols = len(col_seq)-1.
"""
input_matrix = input_matrix.transpose()
output_matrix = self._MaxPoolAlongRows(input_matrix, col_seq, overlapping)
return output_matrix.transpose()
def _GetExpectedFractionalMaxPoolResult(self, input_tensor, row_seq, col_seq,
overlapping):
"""Get expected fractional max pool result.
row_seq and col_seq together defines the fractional pooling region.
Args:
input_tensor: Original input tensor, assuming it is a 4-D tensor, with
dimension as [batch, height/row, width/column, channels/depth].
row_seq: Cumulative pooling sequence along row.
col_seq: Cumulative pooling sequence along column.
overlapping: Use overlapping when doing pooling.
Returns:
A 4-D tensor that is the result of max pooling on input_tensor based on
pooling region defined by row_seq and col_seq, conditioned on whether or
not overlapping is used.
"""
input_shape = input_tensor.shape
output_shape = (input_shape[0], len(row_seq) - 1, len(col_seq) - 1,
input_shape[3])
output_tensor = np.zeros(shape=output_shape, dtype=input_tensor.dtype)
for batch in range(input_shape[0]):
for channel in range(input_shape[3]):
two_dim_slice = input_tensor[batch, :, :, channel]
tmp = self._MaxPoolAlongRows(two_dim_slice, row_seq, overlapping)
output_tensor[batch, :, :, channel] = self._MaxPoolAlongCols(
tmp, col_seq, overlapping)
return output_tensor
def _ValidateFractionalMaxPoolResult(self, input_tensor, pooling_ratio,
pseudo_random, overlapping):
"""Validate FractionalMaxPool's result against expected.
Expected result is computed given input_tensor, and pooling region defined
by row_seq and col_seq.
Args:
input_tensor: A tensor or numpy ndarray.
pooling_ratio: A list or tuple of length 4, first and last element be 1.
pseudo_random: Use pseudo random method to generate pooling sequence.
overlapping: Use overlapping when pooling.
Returns:
None
"""
with self.cached_session():
p, r, c = nn_ops.fractional_max_pool_v2(
input_tensor,
pooling_ratio,
pseudo_random,
overlapping,
seed=self._SEED)
actual, row_seq, col_seq = self.evaluate([p, r, c])
expected = self._GetExpectedFractionalMaxPoolResult(input_tensor, row_seq,
col_seq, overlapping)
self.assertShapeEqual(expected, p)
self.assertAllClose(expected, actual)
def _testVisually(self):
"""Manual test by printing out intermediate result of a small random tensor.
Since _GetExpectedFractionalMaxPoolResult is 'automated', it feel safer to
have a test case that you can see what's happening.
This test will generate a small, random, int 2D matrix, and feed it to
FractionalMaxPool and _GetExpectedFractionalMaxPoolResult.
"""
num_rows = 6
num_cols = 6
tensor_shape = (1, num_rows, num_cols, 1)
pseudo_random = False
for overlapping in True, False:
print("-" * 70)
print("Testing FractionalMaxPool with overlapping = {}".format(
overlapping))
rand_mat = self._PRNG.randint(10, size=tensor_shape)
pooling_ratio = [1, math.sqrt(2), math.sqrt(2), 1]
with self.cached_session():
p, r, c = nn_ops.fractional_max_pool_v2(
rand_mat,
pooling_ratio,
pseudo_random,
overlapping,
seed=self._SEED)
tensor_output, row_seq, col_seq = self.evaluate([p, r, c])
expected_result = self._GetExpectedFractionalMaxPoolResult(rand_mat,
row_seq,
col_seq,
overlapping)
print("row sequence:")
print(row_seq)
print("column sequence:")
print(col_seq)
print("Input:")
# Print input with pooling region marked.
for i in range(num_rows):
row_to_print = []
for j in range(num_cols):
if j in col_seq:
row_to_print.append("|")
row_to_print.append(str(rand_mat[0, i, j, 0]))
row_to_print.append("|")
if i in row_seq:
print("-" * 2 * len(row_to_print))
print(" ".join(row_to_print))
print("-" * 2 * len(row_to_print))
print("Output from FractionalMaxPool:")
print(tensor_output[0, :, :, 0])
print("Expected result:")
print(expected_result[0, :, :, 0])
def testAllInputOptions(self):
"""Try all possible input options for fractional_max_pool.
"""
num_batches = 5
num_channels = 3
num_rows = 20
num_cols = 30
for pseudo_random in True, False:
for overlapping in True, False:
tensor_shape = (num_batches, num_rows, num_cols, num_channels)
# random tensor with value in [-500.0, 500.0)
rand_mat = self._PRNG.random_sample(tensor_shape) * 1000 - 500
self._ValidateFractionalMaxPoolResult(
rand_mat, [1, math.sqrt(3), math.sqrt(2), 1], pseudo_random,
overlapping)
def testIntegerTensorInput(self):
"""Test it works fine when input tensor is integer type.
"""
num_batches = 5
num_channels = 3
num_rows = 20
num_cols = 30
pseudo_random = True
overlapping = True
tensor_shape = (num_batches, num_rows, num_cols, num_channels)
rand_mat = self._PRNG.randint(1000, size=tensor_shape)
self._ValidateFractionalMaxPoolResult(rand_mat,
[1, math.sqrt(3), math.sqrt(2), 1],
pseudo_random, overlapping)
def testDifferentTensorShapes(self):
"""Test different shapes of input tensor.
Mainly test different combinations of num_rows and num_cols.
"""
pseudo_random = True
overlapping = True
for num_batches in [1, 3]:
for num_channels in [1, 3]:
for num_rows in [10, 20, 50]:
for num_cols in [10, 20, 50]:
tensor_shape = (num_batches, num_rows, num_cols, num_channels)
# random tensor with value in [-500.0, 500.0)
rand_mat = self._PRNG.random_sample(tensor_shape) * 1000 - 500
self._ValidateFractionalMaxPoolResult(
rand_mat, [1, math.sqrt(3), math.sqrt(2), 1], pseudo_random,
overlapping)
def testLargePoolingRatio(self):
"""Test when pooling ratio is not within [1, 2).
"""
pseudo_random = True
overlapping = True
num_batches = 3
num_channels = 3
num_rows = 30
num_cols = 50
tensor_shape = (num_batches, num_rows, num_cols, num_channels)
for row_ratio in [math.sqrt(11), math.sqrt(37)]:
for col_ratio in [math.sqrt(11), math.sqrt(27)]:
# random tensor with value in [-500.0, 500.0)
rand_mat = self._PRNG.random_sample(tensor_shape) * 1000 - 500
self._ValidateFractionalMaxPoolResult(rand_mat,
[1, row_ratio, col_ratio, 1],
pseudo_random, overlapping)
def testDivisiblePoolingRatio(self):
"""Test when num of rows/cols can evenly divide pooling ratio.
This is a case regular max pooling can handle. Should be handled by
fractional pooling as well.
"""
pseudo_random = True
overlapping = True
num_batches = 3
num_channels = 3
num_rows = 30
num_cols = 50
tensor_shape = (num_batches, num_rows, num_cols, num_channels)
# random tensor with value in [-500.0, 500.0)
rand_mat = self._PRNG.random_sample(tensor_shape) * 1000 - 500
self._ValidateFractionalMaxPoolResult(rand_mat, [1, 2, 2, 1], pseudo_random,
overlapping)
@test_util.run_deprecated_v1
def testDifferentInputTensorShape(self):
"""Runs the operation in one session with different input tensor shapes."""
with self.cached_session() as sess:
input_holder = array_ops.placeholder(dtypes.float32,
[None, None, None, 3])
pooling_ratio = [1, 1.5, 1.5, 1]
pseudo_random = False
overlapping = False
p, r, c = nn_ops.fractional_max_pool_v2(
input_holder,
pooling_ratio,
pseudo_random,
overlapping,
seed=self._SEED)
# First run.
input_a = np.zeros([3, 32, 32, 3])
actual, row_seq, col_seq = sess.run([p, r, c], {input_holder: input_a})
expected = self._GetExpectedFractionalMaxPoolResult(
input_a, row_seq, col_seq, overlapping)
self.assertSequenceEqual(expected.shape, actual.shape)
# Second run.
input_b = np.zeros([4, 45, 45, 3])
actual, row_seq, col_seq = sess.run([p, r, c], {input_holder: input_b})
expected = self._GetExpectedFractionalMaxPoolResult(
input_b, row_seq, col_seq, overlapping)
self.assertSequenceEqual(expected.shape, actual.shape)
def testDeterminismExceptionThrowing(self):
tensor_shape = (5, 20, 20, 3)
rand_mat = self._PRNG.random_sample(tensor_shape) * 1000 - 500
with test_util.deterministic_ops():
with self.assertRaisesRegex(
ValueError, "requires a non-zero seed to be passed in when "
"determinism is enabled"):
nn_ops.fractional_max_pool_v2(rand_mat, [1, 1.5, 1.5, 1])
nn_ops.fractional_max_pool_v2(rand_mat, [1, 1.5, 1.5, 1], seed=1)
with self.assertRaisesRegex(ValueError,
'requires "seed" and "seed2" to be non-zero'):
nn_ops.fractional_max_pool(rand_mat, [1, 1.5, 1.5, 1])
nn_ops.fractional_max_pool(
rand_mat, [1, 1.5, 1.5, 1], seed=1, seed2=1, deterministic=True)
def testPoolingRatioHasMoreDimThanInput(self):
with self.cached_session() as _:
with self.assertRaisesRegex(
errors.InvalidArgumentError,
r"Pooling ratio is higher than input dimension size for dimension 1.*"
):
result = nn_ops.gen_nn_ops.fractional_max_pool(
value=constant_op.constant(
value=[[[[1, 4, 2, 3]]]], dtype=dtypes.int64),
pooling_ratio=[1.0, 1.44, 1.73, 1.0],
pseudo_random=False,
overlapping=False,
deterministic=False,
seed=0,
seed2=0,
name=None)
self.evaluate(result)
def testPoolingRatioIllegalSmallValue(self):
with self.cached_session() as _:
# Whether turn on `TF2_BEHAVIOR` generates different error messages
with self.assertRaisesRegex(
(errors.InvalidArgumentError, ValueError),
r"(pooling_ratio cannot be smaller than 1, got: .*)|(is negative)"):
result = nn_ops.gen_nn_ops.fractional_max_pool(
value=np.zeros([3, 30, 30, 3]),
pooling_ratio=[1, -1, 3, 1],
pseudo_random=False,
overlapping=False,
deterministic=False,
seed=0,
seed2=0,
)
self.evaluate(result)
def testPoolingIllegalRatioForBatch(self):
with self.cached_session() as _:
with self.assertRaises(errors.UnimplementedError):
result = nn_ops.fractional_max_pool(
np.zeros([3, 30, 50, 3]),
[2, 3, 1.5, 1],
True,
True)
self.evaluate(result)
class FractionalMaxPoolGradTest(test.TestCase):
"""Tests for FractionalMaxPoolGrad.
Two types of tests for FractionalMaxPoolGrad.
1) Test fractional_max_pool_grad() directly.
This type of test relies on gen_nn_ops.max_pool_grad() returns the correct
result. For example:
* input_tensor_shape = (1, 10, 10, 1)
* window_size = (1, 2, 2, 1)
* stride_size = (1, 2, 2, 1)
* padding: not really import, since 10/2 is divisible
max pooling should generate the same result as fractional max pooling with:
* row_sequence = [0, 2, 4, 6, 8, 10]
* col_sequence = [0, 2, 4, 6, 8, 10]
* overlapping = False
This also means their gradients in such case will be the same.
Similarly, when
* input_tensor_shape = (1, 7, 7, 1)
* window_size = (1, 3, 3, 1)
* stride_size = (1, 2, 2, 1)
* padding: not important
max pooling should generate the same result as fractional max pooling with:
* row_sequence = [0, 2, 4, 7]
* col_sequence = [0, 2, 4, 7]
* overlapping = True
2) Test through compute_gradient_error()
"""
_PRNG = np.random.RandomState(341261)
_SEED = 123456
def _GenerateUniqueRandomInputTensor(self, shape):
"""Generate 'unique' random input tensor.
'Unique' means there's no collision values in the tensor, all elements are
different. This is done by generating sequence of integers with step of 1
and then randomly shuffle these integers.
Args:
shape: Shape of the tensor desired.
Returns:
A numpy ndarray with size = shape and dtype = numpy.float32.
"""
num_elements = 1
for size in shape:
num_elements *= size
x = np.arange(num_elements, dtype=np.float32)
self._PRNG.shuffle(x)
return x.reshape(shape)
def testDirectNotUseOverlapping(self):
for num_batches in [1, 3]:
for row_window_size in [2, 5]:
for col_window_size in [2, 4]:
num_rows = row_window_size * 5
num_cols = col_window_size * 7
for num_channels in [1, 2]:
input_shape = (num_batches, num_rows, num_cols, num_channels)
with self.cached_session() as _:
input_tensor = constant_op.constant(
self._GenerateUniqueRandomInputTensor(input_shape))
window_size = [1, row_window_size, col_window_size, 1]
stride_size = [1, row_window_size, col_window_size, 1]
padding = "VALID"
output_tensor = nn_ops.max_pool(input_tensor, window_size,
stride_size, padding)
output_data = self.evaluate(output_tensor)
output_backprop = self._PRNG.randint(100, size=output_data.shape)
input_backprop_tensor = gen_nn_ops.max_pool_grad(
input_tensor, output_tensor, output_backprop, window_size,
stride_size, padding)
input_backprop = self.evaluate(input_backprop_tensor)
row_seq = list(range(0, num_rows + 1, row_window_size))
col_seq = list(range(0, num_cols + 1, col_window_size))
fmp_input_backprop_tensor = gen_nn_ops.fractional_max_pool_grad(
input_tensor,
output_tensor,
output_backprop,
row_seq,
col_seq,
overlapping=False)
fmp_input_backprop = self.evaluate(fmp_input_backprop_tensor)
self.assertShapeEqual(input_backprop, fmp_input_backprop_tensor)
self.assertAllClose(input_backprop, fmp_input_backprop)
def testDirectUseOverlapping(self):
for num_batches in [1, 3]:
for row_window_size in [2, 5]:
for col_window_size in [2, 4]:
num_rows = (row_window_size - 1) * 5 + 1
num_cols = (col_window_size - 1) * 7 + 1
for num_channels in [1, 2]:
input_shape = (num_batches, num_rows, num_cols, num_channels)
with self.cached_session() as _:
input_tensor = constant_op.constant(
self._GenerateUniqueRandomInputTensor(input_shape))
window_size = [1, row_window_size, col_window_size, 1]
stride_size = [1, row_window_size - 1, col_window_size - 1, 1]
padding = "VALID"
output_tensor = nn_ops.max_pool(input_tensor, window_size,
stride_size, padding)
output_data = self.evaluate(output_tensor)
output_backprop = self._PRNG.randint(100, size=output_data.shape)
input_backprop_tensor = gen_nn_ops.max_pool_grad(
input_tensor, output_tensor, output_backprop, window_size,
stride_size, padding)
input_backprop = self.evaluate(input_backprop_tensor)
row_seq = list(range(0, num_rows, row_window_size - 1))
col_seq = list(range(0, num_cols, col_window_size - 1))
row_seq[-1] += 1
col_seq[-1] += 1
fmp_input_backprop_tensor = gen_nn_ops.fractional_max_pool_grad(
input_tensor,
output_tensor,
output_backprop,
row_seq,
col_seq,
overlapping=True)
fmp_input_backprop = self.evaluate(fmp_input_backprop_tensor)
self.assertShapeEqual(input_backprop, fmp_input_backprop_tensor)
self.assertAllClose(input_backprop, fmp_input_backprop)
@test_util.run_deprecated_v1
def testAllInputOptionsThroughGradientError(self):
input_shape = (1, 7, 13, 1)
input_data = self._GenerateUniqueRandomInputTensor(input_shape)
# Add some randomness to make input_data not so 'integer'
input_data += self._PRNG.random_sample(input_shape)
pooling_ratio = [1, math.sqrt(2), math.sqrt(3), 1]
for pseudo_random in True, False:
for overlapping in True, False:
with self.cached_session() as _:
input_tensor = constant_op.constant(input_data, shape=input_shape)
output_tensor, unused_a, unused_b = nn_ops.fractional_max_pool_v2(
input_tensor,
pooling_ratio,
pseudo_random=pseudo_random,
overlapping=overlapping,
seed=self._SEED)
output_data = self.evaluate(output_tensor)
output_shape = output_data.shape
# error_margin and delta setting is similar to max_pool_grad.
error_margin = 1e-3
gradient_error = gradient_checker.compute_gradient_error(
input_tensor,
input_shape,
output_tensor,
output_shape,
x_init_value=input_data.reshape(input_shape),
delta=1e-2)
self.assertLess(gradient_error, error_margin)
@test_util.run_deprecated_v1
def testDifferentTensorShapesThroughGradientError(self):
pseudo_random = True
overlapping = True
pooling_ratio = [1, math.sqrt(3), math.sqrt(2), 1]
for num_batches in [1, 2]:
for num_rows in [5, 13]:
for num_cols in [5, 11]:
for num_channels in [1, 3]:
input_shape = (num_batches, num_rows, num_cols, num_channels)
input_data = self._GenerateUniqueRandomInputTensor(input_shape)
# Add some randomness to make input_data not so 'integer'
input_data += self._PRNG.random_sample(input_shape)
with self.cached_session() as _:
input_tensor = constant_op.constant(input_data, shape=input_shape)
output_tensor, unused_a, unused_b = nn_ops.fractional_max_pool_v2(
input_tensor,
pooling_ratio,
pseudo_random=pseudo_random,
overlapping=overlapping,
seed=self._SEED)
output_data = self.evaluate(output_tensor)
output_shape = output_data.shape
# error_margin and delta setting is similar to max_pool_grad.
error_margin = 1e-3
gradient_error = gradient_checker.compute_gradient_error(
input_tensor,
input_shape,
output_tensor,
output_shape,
x_init_value=input_data.reshape(input_shape),
delta=1e-2)
self.assertLess(gradient_error, error_margin)
@test_util.run_deprecated_v1
def testLargePoolingRatioThroughGradientError(self):
input_shape = (1, 17, 23, 1)
input_data = self._GenerateUniqueRandomInputTensor(input_shape)
# Add some randomness to make input_data not so 'integer'
input_data += self._PRNG.random_sample(input_shape)
pooling_ratio = (1, math.sqrt(13), math.sqrt(7), 1)
output_shape = [int(a / b) for a, b in zip(input_shape, pooling_ratio)]
overlapping = True
pseudo_random = False
with self.cached_session() as _:
input_tensor = constant_op.constant(input_data, shape=input_shape)
output_tensor, unused_a, unused_b = nn_ops.fractional_max_pool_v2(
input_tensor,
pooling_ratio,
pseudo_random=pseudo_random,
overlapping=overlapping,
seed=self._SEED)
# error_margin and delta setting is similar to max_pool_grad.
error_margin = 1e-3
gradient_error = gradient_checker.compute_gradient_error(
input_tensor,
input_shape,
output_tensor,
output_shape,
x_init_value=input_data.reshape(input_shape),
delta=1e-2)
self.assertLess(gradient_error, error_margin)
def testWhenRepeatedMaxValueInPoolingRegion(self):
"""Test when there's repeating value in pooling region.
There's no formal definition for what the gradient should be when there're
multiple max value within a pooling cell. Such as
| 1 5 |
| 5 3 |
The expected result depends heavily on implementation, if someone swap the
order of a nested for loop when walking through the tensor, result would be
very different.
The goal of this test is to alert when someone else change the
implementation. Current implementation scans row-by-row.
"""
input_data = [5.0, 4.0, 6.0, 7.0,
3.0, 5.0, 9.0, 6.0,
8.0, 8.0, 9.0, 5.0,
7.0, 4.0, 0.0, 0.0] # pyformat: disable
input_size = [1, 4, 4, 1]
output_backprop = [12.0, 15.0,
17.0, -5.0,
6.0, 21.0] # pyformat: disable
row_seq = [0, 1, 3, 4]
col_seq = [0, 2, 4]
output_data_not_overlapping = [5.0, 7.0,
8.0, 9.0,
7.0, 0.0] # pyformat: disable
output_data_overlapping = [9.0, 9.0,
9.0, 9.0,
7.0, 0.0] # pyformat: disable
output_size = [1, 3, 2, 1]
expected_input_backprop_not_overlapping = np.reshape(
[12.0, 0.0, 0.0, 15.0,
0.0, 0.0, -5.0, 0.0,
17.0, 0.0, 0.0, 0.0,
6.0, 0.0, 21.0, 0.0],
input_size) # pyformat: disable
expected_input_backprop_overlapping = np.reshape(
[0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 39.0, 0.0,
0.0, 0.0, 0.0, 0.0,
6.0, 0.0, 21.0, 0.0],
input_size) # pyformat: disable
with self.cached_session() as _:
# Test when overlapping is False
input_tensor = constant_op.constant(input_data, shape=input_size)
output_tensor = constant_op.constant(
output_data_not_overlapping, shape=output_size)
grad = constant_op.constant(output_backprop, shape=output_size)
r = gen_nn_ops.fractional_max_pool_grad(
input_tensor,
output_tensor,
grad,
row_seq,
col_seq,
overlapping=False)
input_backprop_not_overlapping = self.evaluate(r)
self.assertShapeEqual(
np.reshape(expected_input_backprop_not_overlapping, input_size), r)
self.assertAllClose(expected_input_backprop_not_overlapping,
input_backprop_not_overlapping)
# Test when overlapping is True
output_tensor = constant_op.constant(
output_data_overlapping, shape=output_size)
r = gen_nn_ops.fractional_max_pool_grad(
input_tensor, output_tensor, grad, row_seq, col_seq, overlapping=True)
input_backprop_overlapping = self.evaluate(r)
self.assertShapeEqual(
np.reshape(expected_input_backprop_overlapping, input_size), r)
self.assertAllClose(expected_input_backprop_overlapping,
input_backprop_overlapping)
def testInvalidSeqRaiseErrorForFractionalMaxPoolGrad(self):
with self.assertRaises(errors.InvalidArgumentError):
with self.cached_session():
overlapping = True
orig_input = constant_op.constant(
.453409232, shape=[1, 7, 13, 1], dtype=dtypes.float32)
orig_output = constant_op.constant(
.453409232, shape=[1, 7, 13, 1], dtype=dtypes.float32)
out_backprop = constant_op.constant(
.453409232, shape=[1, 7, 13, 1], dtype=dtypes.float32)
row_pooling_sequence = constant_op.constant(
0, shape=[5], dtype=dtypes.int64)
col_pooling_sequence = constant_op.constant(
0, shape=[5], dtype=dtypes.int64)
t = gen_nn_ops.FractionalMaxPoolGrad(
orig_input=orig_input,
orig_output=orig_output,
out_backprop=out_backprop,
row_pooling_sequence=row_pooling_sequence,
col_pooling_sequence=col_pooling_sequence,
overlapping=overlapping)
self.evaluate(t)
def testEmptySeqRaisesErrorForFractionalMaxPoolGrad(self):
with self.assertRaisesRegex(
errors.InvalidArgumentError, "must be a vector"
):
overlapping = True
orig_input = constant_op.constant(
0.453409232, shape=[1, 7, 13, 1], dtype=dtypes.float32
)
orig_output = constant_op.constant(
0.453409232, shape=[1, 7, 13, 1], dtype=dtypes.float32
)
out_backprop = constant_op.constant(
0.453409232, shape=[1, 7, 13, 1], dtype=dtypes.float32
)
row_pooling_sequence = constant_op.constant(
0, shape=[], dtype=dtypes.int64
)
col_pooling_sequence = constant_op.constant(
0, shape=[], dtype=dtypes.int64
)
t = gen_nn_ops.FractionalMaxPoolGrad(
orig_input=orig_input,
orig_output=orig_output,
out_backprop=out_backprop,
row_pooling_sequence=row_pooling_sequence,
col_pooling_sequence=col_pooling_sequence,
overlapping=overlapping,
)
self.evaluate(t)
def testOverLargeSeqRaiseErrorForFractionalMaxPoolGrad(self):
with self.assertRaises(errors.InvalidArgumentError):
with self.cached_session():
overlapping = False
orig_input = [[[[1, 1, 1, 1, 1]]]]
orig_output = [[[[1, 1, 1]]]]
out_backprop = [[[[3], [3], [6]]]]
row_pooling_sequence = [-0x4000000, 1, 1]
col_pooling_sequence = [-0x4000000, 1, 1]
t = gen_nn_ops.FractionalMaxPoolGrad(
orig_input=orig_input,
orig_output=orig_output,
out_backprop=out_backprop,
row_pooling_sequence=row_pooling_sequence,
col_pooling_sequence=col_pooling_sequence,
overlapping=overlapping)
self.evaluate(t)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/losses_test.py
+1578
View File
File diff suppressed because it is too large Load Diff
tensorflow/python/kernel_tests/nn_ops/lrn_op_test.py
+196
View File
@@ -0,0 +1,196 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for local response normalization."""
import copy
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import gradients_impl
from tensorflow.python.ops import nn
from tensorflow.python.ops import random_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
class LRNOpTest(test.TestCase):
def _LRN(self, input_image, lrn_depth_radius=5, bias=1.0, alpha=1.0,
beta=0.5):
"""Compute expected result."""
output = copy.deepcopy(input_image)
batch_size = input_image.shape[0]
rows = input_image.shape[1]
cols = input_image.shape[2]
depth = input_image.shape[3]
for b in range(batch_size):
for r in range(rows):
for c in range(cols):
for d in range(depth):
begin = max(0, d - lrn_depth_radius)
end = min(depth, d + lrn_depth_radius + 1)
patch = input_image[b, r, c, begin:end]
output[b, r, c, d] /= (
np.power(bias + alpha * np.sum(patch * patch), beta))
return output
def _RunAndVerify(self, dtype):
with self.cached_session():
# random shape
shape = np.random.randint(1, 16, size=4)
# Make depth at least 2 to make it meaningful
shape[3] += 1
p = array_ops.placeholder(dtype, shape=shape)
# random depth_radius, bias, alpha, beta. cuDNN requires depth_radius to
# be in [1, 7].
lrn_depth_radius = np.random.randint(1, min(8, shape[3]))
bias = 1.0 + np.random.rand()
alpha = 2.0 * np.random.rand()
# cuDNN requires beta >= 0.01.
beta = 0.01 + 2.0 * np.random.rand()
lrn_t = nn.local_response_normalization(
p,
name="lrn",
depth_radius=lrn_depth_radius,
bias=bias,
alpha=alpha,
beta=beta)
params = {p: np.random.rand(*shape).astype("f")}
result = lrn_t.eval(feed_dict=params)
expected = self._LRN(
params[p],
lrn_depth_radius=lrn_depth_radius,
bias=bias,
alpha=alpha,
beta=beta)
err = np.amax(np.abs(result - expected))
print("LRN error for bias ", bias, "alpha ", alpha, " beta ", beta, " is ",
err)
if dtype == dtypes.float32:
self.assertTrue(err < 1e-4)
else:
self.assertTrue(err < 1e-2)
self.assertShapeEqual(expected, lrn_t)
@test_util.run_deprecated_v1
def testCompute(self):
for _ in range(2):
self._RunAndVerify(dtypes.float32)
# Enable when LRN supports tf.float16 on GPU.
if not test.is_gpu_available():
self._RunAndVerify(dtypes.float16)
@test_util.run_deprecated_v1
def testGradientsZeroInput(self):
with self.session():
shape = [4, 4, 4, 4]
p = array_ops.placeholder(dtypes.float32, shape=shape)
inp_array = np.zeros(shape).astype("f")
lrn_op = nn.local_response_normalization(p, 2, 1.0, 0.0, 1.0, name="lrn")
grad = gradients_impl.gradients([lrn_op], [p])[0]
params = {p: inp_array}
r = grad.eval(feed_dict=params)
expected = np.ones(shape).astype("f")
self.assertAllClose(r, expected)
self.assertShapeEqual(expected, grad)
@test_util.run_in_graph_and_eager_modes
def testIncompatibleInputAndOutputImageShapes(self):
depth_radius = 1
bias = 1.59018219
alpha = 0.117728651
beta = 0.404427052
input_grads = random_ops.random_uniform(
shape=[4, 4, 4, 4],
minval=-10000,
maxval=10000,
dtype=dtypes.float32,
seed=-2033)
input_image = random_ops.random_uniform(
shape=[4, 4, 4, 4],
minval=-10000,
maxval=10000,
dtype=dtypes.float32,
seed=-2033)
invalid_output_image = random_ops.random_uniform(
shape=[4, 4, 4, 4, 4, 4],
minval=-10000,
maxval=10000,
dtype=dtypes.float32,
seed=-2033)
with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)):
self.evaluate(
nn.lrn_grad(
input_grads=input_grads,
input_image=input_image,
output_image=invalid_output_image,
depth_radius=depth_radius,
bias=bias,
alpha=alpha,
beta=beta))
def _RunAndVerifyGradients(self, dtype):
with self.cached_session():
# random shape
shape = np.random.randint(1, 5, size=4)
# Make depth at least 2 to make it meaningful
shape[3] += 1
# random depth_radius, bias, alpha, beta. cuDNN requires depth_radius to
# be in [1, 7].
lrn_depth_radius = np.random.randint(1, min(8, shape[3]))
bias = 1.0 + np.random.rand()
alpha = 1.0 * np.random.rand()
# cuDNN requires beta >= 0.01.
beta = 0.01 + 1.0 * np.random.rand()
if dtype == dtypes.float32:
inp_array = np.random.rand(*shape).astype(np.float32)
else:
inp_array = np.random.rand(*shape).astype(np.float16)
inp = constant_op.constant(
list(inp_array.ravel(order="C")), shape=shape, dtype=dtype)
lrn_op = nn.local_response_normalization(
inp,
name="lrn",
depth_radius=lrn_depth_radius,
bias=bias,
alpha=alpha,
beta=beta)
err = gradient_checker.compute_gradient_error(inp, shape, lrn_op, shape)
print("LRN Gradient error for bias ", bias, "alpha ", alpha, " beta ", beta,
" is ", err)
if dtype == dtypes.float32:
self.assertLess(err, 1e-4)
else:
self.assertLess(err, 1.0)
@test_util.run_deprecated_v1
def testGradients(self):
for _ in range(2):
self._RunAndVerifyGradients(dtypes.float32)
# Enable when LRN supports tf.float16 on GPU.
if not test.is_gpu_available():
self._RunAndVerifyGradients(dtypes.float16)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/morphological_ops_test.py
+643
View File
@@ -0,0 +1,643 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Functional tests for morphological filtering operations."""
from absl.testing import parameterized
import numpy as np
from tensorflow.python.framework import config
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import test_util
from tensorflow.python.ops import gradient_checker_v2
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
class DilationTest(test.TestCase, parameterized.TestCase):
def _VerifyValues(self, image, kernel, strides, rates, padding, out, use_gpu,
dtype):
"""Verifies the output values of the dilation function.
Args:
image: Input tensor with shape: [batch, in_height, in_width, channels].
kernel: Filter tensor with shape: [filter_height, filter_width, channels].
strides: Output strides, specified as [stride_height, stride_width].
rates: Atrous rates, specified as [rate_height, rate_width].
padding: Padding type.
out: Expected output.
use_gpu: Whether we are running on GPU.
"""
strides = [1] + strides + [1]
rates = [1] + rates + [1]
with self.cached_session(use_gpu=use_gpu):
out_tensor = nn_ops.dilation2d(
constant_op.constant(image, dtype=dtype),
constant_op.constant(kernel, dtype=dtype),
strides=strides,
rates=rates,
padding=padding,
name="dilation2d")
self.assertAllCloseAccordingToType(out, self.evaluate(out_tensor))
def _testDilationValidPadding(self, use_gpu, dtype):
# [1, 2, 2, 1]
image = [[[[.1], [.2]], [[.3], [.4]]]]
# [2, 2, 1]
kernel = [[[.4], [.3]], [[.1], [.0]]]
# [1, 1, 1, 1]
out = [[[[.5]]]]
self._VerifyValues(
image,
kernel,
strides=[1, 1],
rates=[1, 1],
padding="VALID",
out=out,
use_gpu=use_gpu,
dtype=dtype)
def _testDilationSamePadding(self, use_gpu, dtype):
# [1, 2, 2, 1]
image = [[[[.1], [.2]], [[.3], [.4]]]]
# [2, 2, 1]
kernel = [[[.4], [.3]], [[.1], [.0]]]
# [1, 2, 2, 1]
out = [[[[.5], [.6]], [[.7], [.8]]]]
self._VerifyValues(
image,
kernel,
strides=[1, 1],
rates=[1, 1],
padding="SAME",
out=out,
use_gpu=use_gpu,
dtype=dtype)
def _testDilationSamePaddingDepth(self, use_gpu, dtype):
# [1, 2, 2, 3]
image = [[[[.1, .2, .0], [.2, .3, .1]], [[.3, .4, .2], [.4, .5, .3]]]]
# [2, 2, 3]
kernel = [[[.4, .5, .3], [.3, .4, .2]], [[.1, .2, .0], [.0, .1, -.1]]]
# [1, 2, 2, 3]
out = [[[[.5, .7, .3], [.6, .8, .4]], [[.7, .9, .5], [.8, 1., .6]]]]
self._VerifyValues(
image,
kernel,
strides=[1, 1],
rates=[1, 1],
padding="SAME",
out=out,
use_gpu=use_gpu,
dtype=dtype)
def _testDilationSamePaddingBatch(self, use_gpu, dtype):
# [2, 2, 2, 1]
image = [[[[.1], [.2]], [[.3], [.4]]], [[[.2], [.3]], [[.4], [.5]]]]
# [2, 2, 1]
kernel = [[[.4], [.3]], [[.1], [.0]]]
# [2, 2, 2, 1]
out = [[[[.5], [.6]], [[.7], [.8]]], [[[.6], [.7]], [[.8], [.9]]]]
self._VerifyValues(
image,
kernel,
strides=[1, 1],
rates=[1, 1],
padding="SAME",
out=out,
use_gpu=use_gpu,
dtype=dtype)
def _testDilationValidPaddingNonSquareWindow(self, use_gpu, dtype):
# [1, 2, 2, 1]
image = [[[[.1], [.2]], [[.3], [.4]]]]
# [1, 2, 1]
kernel = [[[.4], [.3]]]
# [1, 2, 1, 1]
out = [[[[.5]], [[.7]]]]
self._VerifyValues(
image,
kernel,
strides=[1, 1],
rates=[1, 1],
padding="VALID",
out=out,
use_gpu=use_gpu,
dtype=dtype)
def _testDilationSamePaddingRate(self, use_gpu, dtype):
# [1, 3, 3, 1]
image = [[[[.1], [.2], [.3]], [[.4], [.5], [.6]], [[.7], [.8], [.9]]]]
# [2, 2, 1]
kernel = [[[.4], [.3]], [[.1], [.2]]]
# Because rate = 2.0, the effective kernel is [3, 3, 1]:
# kernel_eff = [[[.4], [.0], [.3]],
# [[.0], [.0], [.0]],
# [[.1], [.0], [.2]]]
# [1, 3, 3, 1]
out = [[[[.7], [.8], [.6]], [[1.0], [1.1], [.9]], [[.8], [.9], [.9]]]]
self._VerifyValues(
image,
kernel,
strides=[1, 1],
rates=[2, 2],
padding="SAME",
out=out,
use_gpu=use_gpu,
dtype=dtype)
def _testDilationValidPaddingUnevenStride(self, use_gpu, dtype):
# [1, 3, 3, 1]
image = [[[[.1], [.2], [.3], [.4]], [[.5], [.6], [.7], [.8]],
[[.9], [1.0], [1.1], [1.2]]]]
# [2, 2, 1]
kernel = [[[.4], [.3]], [[.1], [.2]]]
# [1, 2, 2, 1]
out = [[[[.8], [1.0]], [[1.2], [1.4]]]]
self._VerifyValues(
image,
kernel,
strides=[1, 2],
rates=[1, 1],
padding="VALID",
out=out,
use_gpu=use_gpu,
dtype=dtype)
@parameterized.parameters(dtypes.float32, dtypes.bfloat16)
def testDilation(self, dtype):
for use_gpu in True, False:
self._testDilationValidPadding(use_gpu, dtype)
self._testDilationSamePadding(use_gpu, dtype)
self._testDilationSamePaddingDepth(use_gpu, dtype)
self._testDilationSamePaddingBatch(use_gpu, dtype)
self._testDilationValidPaddingNonSquareWindow(use_gpu, dtype)
self._testDilationSamePaddingRate(use_gpu, dtype)
self._testDilationValidPaddingUnevenStride(use_gpu, dtype)
def _ConstructAndTestGradient(self,
image_shape,
kernel_shape,
strides,
rates,
padding,
use_gpu,
dtype=dtypes.float32):
"""Verifies the gradients of the dilation function.
Args:
image_shape: Input shape, [batch, in_height, in_width, channels].
kernel_shape: Filter shape, [filter_height, filter_width, channels].
strides: Output strides, specified as [stride_height, stride_width].
rates: Atrous rates, specified as [rate_height, rate_width].
padding: Padding type.
use_gpu: Whether we are running on GPU.
"""
assert image_shape[3] == kernel_shape[2]
np.random.seed(1) # Make it reproducible.
image = np.random.random_sample(image_shape).astype(np.float32)
kernel = np.random.random_sample(kernel_shape).astype(np.float32)
strides = [1] + strides + [1]
rates = [1] + rates + [1]
image_tensor = constant_op.constant(
image, shape=image_shape, name="input", dtype=dtype)
kernel_tensor = constant_op.constant(
kernel, shape=kernel_shape, name="filter", dtype=dtype)
def compute_dilation2d(image_tensor, kernel_tensor):
return nn_ops.dilation2d(
image_tensor,
kernel_tensor,
strides=strides,
rates=rates,
padding=padding,
name="dilation2d")
with test_util.device(use_gpu=use_gpu):
with self.cached_session():
# Small delta is necessary for argmax to remain the same.
err1 = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(
lambda x: compute_dilation2d(x, kernel_tensor), [image_tensor]))
err2 = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(
lambda x: compute_dilation2d(image_tensor, x), [kernel_tensor]))
err = max(err1, err2)
print("Dilation gradient error = %f" % err)
if dtype == dtypes.bfloat16:
self.assertLess(err, 4.0)
else:
self.assertLess(err, 1e-4)
def _testDilationGradValidPadding_1x1x1(self, use_gpu, dtype):
self._ConstructAndTestGradient(
image_shape=[1, 3, 3, 1],
kernel_shape=[1, 1, 1],
strides=[1, 1],
rates=[1, 1],
padding="VALID",
use_gpu=use_gpu,
dtype=dtype)
def _testDilationGradDeterminismError(self, use_gpu, dtype):
if use_gpu and test.is_gpu_available(cuda_only=True):
try:
config.enable_op_determinism()
with self.assertRaisesRegex(
errors_impl.UnimplementedError, "Determinism is not yet supported "
"for Dilation2DBackpropInput."):
self._ConstructAndTestGradient(
image_shape=[1, 3, 3, 1],
kernel_shape=[1, 1, 1],
strides=[1, 1],
rates=[1, 1],
padding="VALID",
use_gpu=use_gpu,
dtype=dtype)
finally:
config.disable_op_determinism()
else:
try:
config.enable_op_determinism()
self._ConstructAndTestGradient(
image_shape=[1, 3, 3, 1],
kernel_shape=[1, 1, 1],
strides=[1, 1],
rates=[1, 1],
padding="VALID",
use_gpu=use_gpu,
dtype=dtype)
finally:
config.disable_op_determinism()
def _testDilationGradSamePadding_1x1x1(self, use_gpu, dtype):
self._ConstructAndTestGradient(
image_shape=[1, 3, 3, 1],
kernel_shape=[1, 1, 1],
strides=[1, 1],
rates=[1, 1],
padding="SAME",
use_gpu=use_gpu,
dtype=dtype)
def _testDilationGradSamePadding_1x1x2(self, use_gpu, dtype):
self._ConstructAndTestGradient(
image_shape=[1, 3, 3, 2],
kernel_shape=[1, 1, 2],
strides=[1, 1],
rates=[1, 1],
padding="SAME",
use_gpu=use_gpu,
dtype=dtype)
def _testDilationGradValidPadding_2x2x1(self, use_gpu, dtype):
self._ConstructAndTestGradient(
image_shape=[1, 3, 3, 1],
kernel_shape=[2, 2, 1],
strides=[1, 1],
rates=[1, 1],
padding="VALID",
use_gpu=use_gpu,
dtype=dtype)
def _testDilationGradSamePadding_2x2x1(self, use_gpu, dtype):
self._ConstructAndTestGradient(
image_shape=[1, 3, 3, 1],
kernel_shape=[2, 2, 1],
strides=[1, 1],
rates=[1, 1],
padding="SAME",
use_gpu=use_gpu,
dtype=dtype)
def _testDilationGradSamePaddingBatch_2x2x1(self, use_gpu, dtype):
self._ConstructAndTestGradient(
image_shape=[4, 3, 3, 1],
kernel_shape=[2, 2, 1],
strides=[1, 1],
rates=[1, 1],
padding="SAME",
use_gpu=use_gpu,
dtype=dtype)
def _testDilationGradSamePadding_2x2x4(self, use_gpu, dtype):
self._ConstructAndTestGradient(
image_shape=[1, 3, 3, 4],
kernel_shape=[2, 2, 4],
strides=[1, 1],
rates=[1, 1],
padding="SAME",
use_gpu=use_gpu,
dtype=dtype)
@parameterized.parameters(dtypes.float32, dtypes.bfloat16)
def testDilationGrad(self, dtype):
for use_gpu in True, False:
self._testDilationGradDeterminismError(use_gpu, dtype)
self._testDilationGradValidPadding_1x1x1(use_gpu, dtype)
self._testDilationGradSamePadding_1x1x1(use_gpu, dtype)
self._testDilationGradSamePadding_1x1x2(use_gpu, dtype)
self._testDilationGradValidPadding_2x2x1(use_gpu, dtype)
self._testDilationGradSamePadding_2x2x1(use_gpu, dtype)
self._testDilationGradSamePaddingBatch_2x2x1(use_gpu, dtype)
self._testDilationGradSamePadding_2x2x4(use_gpu, dtype)
class ErosionTest(test.TestCase):
def _VerifyValues(self, image, kernel, strides, rates, padding, out, use_gpu):
"""Verifies the output values of the erosion function.
Args:
image: Input tensor with shape: [batch, in_height, in_width, channels].
kernel: Filter tensor with shape: [filter_height, filter_width, channels].
strides: Output strides, specified as [stride_height, stride_width].
rates: Atrous rates, specified as [rate_height, rate_width].
padding: Padding type.
out: Expected output.
use_gpu: Whether we are running on GPU.
"""
strides = [1] + strides + [1]
rates = [1] + rates + [1]
with self.cached_session(use_gpu=use_gpu):
out_tensor = nn_ops.erosion2d(
constant_op.constant(image),
constant_op.constant(kernel),
strides=strides,
rates=rates,
padding=padding,
name="erosion2d")
self.assertAllClose(out, self.evaluate(out_tensor))
def _testErosionValidPadding(self, use_gpu):
# [1, 2, 2, 1]
image = [[[[.1], [.2]], [[.3], [.4]]]]
# [2, 2, 1]
kernel = [[[.4], [.3]], [[.1], [.0]]]
# [1, 1, 1, 1]
out = [[[[.0]]]]
self._VerifyValues(
image,
kernel,
strides=[1, 1],
rates=[1, 1],
padding="VALID",
out=out,
use_gpu=use_gpu)
def _testErosionSamePadding(self, use_gpu):
# [1, 2, 2, 1]
image = [[[[.1], [.2]], [[.3], [.4]]]]
# [2, 2, 1]
kernel = [[[.4], [.3]], [[.1], [.0]]]
# [1, 2, 2, 1]
out = [[[[.0], [.1]], [[.3], [.4]]]]
self._VerifyValues(
image,
kernel,
strides=[1, 1],
rates=[1, 1],
padding="SAME",
out=out,
use_gpu=use_gpu)
def _testErosionSamePaddingDepth(self, use_gpu):
# [1, 2, 2, 3]
image = [[[[.1, .2, .0], [.2, .3, .1]], [[.3, .4, .2], [.4, .5, .3]]]]
# [2, 2, 3]
kernel = [[[.4, .5, .3], [.3, .4, .2]], [[.1, .2, .0], [.0, .1, -.1]]]
# [1, 2, 2, 3]
out = [[[[.0, .0, .0], [.1, .1, .1]], [[.3, .3, .3], [.4, .4, .4]]]]
self._VerifyValues(
image,
kernel,
strides=[1, 1],
rates=[1, 1],
padding="SAME",
out=out,
use_gpu=use_gpu)
def _testErosionSamePaddingBatch(self, use_gpu):
# [2, 2, 2, 1]
image = [[[[.1], [.2]], [[.3], [.4]]], [[[.2], [.3]], [[.4], [.5]]]]
# [2, 2, 1]
kernel = [[[.4], [.3]], [[.1], [.0]]]
# [2, 2, 2, 1]
out = [[[[.0], [.1]], [[.3], [.4]]], [[[.1], [.2]], [[.4], [.5]]]]
self._VerifyValues(
image,
kernel,
strides=[1, 1],
rates=[1, 1],
padding="SAME",
out=out,
use_gpu=use_gpu)
def _testErosionValidPaddingNonSquareWindow(self, use_gpu):
# [1, 2, 2, 1]
image = [[[[.1], [.2]], [[.3], [.4]]]]
# [1, 2, 1]
kernel = [[[.4], [.3]]]
# [1, 2, 1, 1]
out = [[[[-.2]], [[.0]]]]
self._VerifyValues(
image,
kernel,
strides=[1, 1],
rates=[1, 1],
padding="VALID",
out=out,
use_gpu=use_gpu)
def _testErosionSamePaddingRate(self, use_gpu):
# [1, 3, 3, 1]
image = [[[[.1], [.2], [.3]], [[.4], [.5], [.6]], [[.7], [.8], [.9]]]]
# [2, 2, 1]
kernel = [[[.4], [.3]], [[.1], [.2]]]
# Because rate = 2.0, the effective kernel is [3, 3, 1]:
# kernel_eff = [[[.4], [.0], [.3]],
# [[.0], [.0], [.0]],
# [[.1], [.0], [.2]]]
# [1, 3, 3, 1]
out = [[[[.1], [.1], [.2]], [[0.1], [-.1], [.0]], [[.4], [.2], [.3]]]]
self._VerifyValues(
image,
kernel,
strides=[1, 1],
rates=[2, 2],
padding="SAME",
out=out,
use_gpu=use_gpu)
def _testErosionValidPaddingUnevenStride(self, use_gpu):
# [1, 3, 3, 1]
image = [[[[.1], [.2], [.3], [.4]], [[.5], [.6], [.7], [.8]],
[[.9], [1.0], [1.1], [1.2]]]]
# [2, 2, 1]
kernel = [[[.4], [.3]], [[.1], [.2]]]
# [1, 2, 2, 1]
out = [[[[-.1], [.1]], [[.3], [.5]]]]
self._VerifyValues(
image,
kernel,
strides=[1, 2],
rates=[1, 1],
padding="VALID",
out=out,
use_gpu=use_gpu)
def testErosion(self):
for use_gpu in True, False:
self._testErosionValidPadding(use_gpu)
self._testErosionSamePadding(use_gpu)
self._testErosionSamePaddingDepth(use_gpu)
self._testErosionSamePaddingBatch(use_gpu)
self._testErosionValidPaddingNonSquareWindow(use_gpu)
self._testErosionSamePaddingRate(use_gpu)
self._testErosionValidPaddingUnevenStride(use_gpu)
def _ConstructAndTestGradient(self, image_shape, kernel_shape, strides, rates,
padding, use_gpu):
"""Verifies the gradients of the erosion function.
Args:
image_shape: Input shape, [batch, in_height, in_width, channels].
kernel_shape: Filter shape, [filter_height, filter_width, channels].
strides: Output strides, specified as [stride_height, stride_width].
rates: Atrous rates, specified as [rate_height, rate_width].
padding: Padding type.
use_gpu: Whether we are running on GPU.
"""
assert image_shape[3] == kernel_shape[2]
np.random.seed(1) # Make it reproducible.
image = np.random.random_sample(image_shape).astype(np.float32)
kernel = np.random.random_sample(kernel_shape).astype(np.float32)
strides = [1] + strides + [1]
rates = [1] + rates + [1]
image_tensor = constant_op.constant(image, shape=image_shape, name="input")
kernel_tensor = constant_op.constant(
kernel, shape=kernel_shape, name="filter")
def compute_erosion2d(image_tensor, kernel_tensor):
return nn_ops.erosion2d(
image_tensor,
kernel_tensor,
strides=strides,
rates=rates,
padding=padding,
name="erosion2d")
with test_util.device(use_gpu=use_gpu):
with self.cached_session():
# Small delta is necessary for argmax to remain the same.
err1 = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(
lambda x: compute_erosion2d(x, kernel_tensor), [image_tensor]))
err2 = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(
lambda x: compute_erosion2d(image_tensor, x), [kernel_tensor]))
err = max(err1, err2)
print("Erosion gradient error = %f" % err)
self.assertLess(err, 1e-4)
def _testErosionGradValidPadding_1x1x1(self, use_gpu):
self._ConstructAndTestGradient(
image_shape=[1, 3, 3, 1],
kernel_shape=[1, 1, 1],
strides=[1, 1],
rates=[1, 1],
padding="VALID",
use_gpu=use_gpu)
def _testErosionGradSamePadding_1x1x1(self, use_gpu):
self._ConstructAndTestGradient(
image_shape=[1, 3, 3, 1],
kernel_shape=[1, 1, 1],
strides=[1, 1],
rates=[1, 1],
padding="SAME",
use_gpu=use_gpu)
def _testErosionGradSamePadding_1x1x2(self, use_gpu):
self._ConstructAndTestGradient(
image_shape=[1, 3, 3, 2],
kernel_shape=[1, 1, 2],
strides=[1, 1],
rates=[1, 1],
padding="SAME",
use_gpu=use_gpu)
def _testErosionGradValidPadding_2x2x1(self, use_gpu):
self._ConstructAndTestGradient(
image_shape=[1, 3, 3, 1],
kernel_shape=[2, 2, 1],
strides=[1, 1],
rates=[1, 1],
padding="VALID",
use_gpu=use_gpu)
def _testErosionGradSamePadding_2x2x1(self, use_gpu):
self._ConstructAndTestGradient(
image_shape=[1, 3, 3, 1],
kernel_shape=[2, 2, 1],
strides=[1, 1],
rates=[1, 1],
padding="SAME",
use_gpu=use_gpu)
def _testErosionGradSamePaddingBatch_2x2x1(self, use_gpu):
self._ConstructAndTestGradient(
image_shape=[4, 3, 3, 1],
kernel_shape=[2, 2, 1],
strides=[1, 1],
rates=[1, 1],
padding="SAME",
use_gpu=use_gpu)
def _testErosionGradSamePadding_2x2x4(self, use_gpu):
self._ConstructAndTestGradient(
image_shape=[1, 3, 3, 4],
kernel_shape=[2, 2, 4],
strides=[1, 1],
rates=[1, 1],
padding="SAME",
use_gpu=use_gpu)
def testErosionGrad(self):
for use_gpu in True, False:
self._testErosionGradValidPadding_1x1x1(use_gpu)
self._testErosionGradSamePadding_1x1x1(use_gpu)
self._testErosionGradSamePadding_1x1x2(use_gpu)
self._testErosionGradValidPadding_2x2x1(use_gpu)
self._testErosionGradSamePadding_2x2x1(use_gpu)
self._testErosionGradSamePaddingBatch_2x2x1(use_gpu)
self._testErosionGradSamePadding_2x2x4(use_gpu)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/nth_element_op_test.py
+188
View File
@@ -0,0 +1,188 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
import numpy as np
from tensorflow.python.eager import backprop
from tensorflow.python.eager import context
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gradients_impl
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
class NthElementTest(test.TestCase):
def _validateNthElement(self, inputs, dtype, n, reverse, expected_values):
np_expected_values = np.array(expected_values)
with self.cached_session(use_gpu=False) as sess:
inputs_op = ops.convert_to_tensor(inputs, dtype=dtype)
values_op = nn_ops.nth_element(inputs_op, n, reverse=reverse)
values = self.evaluate(values_op)
self.assertShapeEqual(np_expected_values, values_op)
self.assertAllClose(np_expected_values, values)
def testExample1(self):
inputs = [2.2, 4.4, 1.1, 5.5, 3.3]
self._validateNthElement(inputs, dtypes.float32, 1, False, 2.2)
self._validateNthElement(inputs, dtypes.float32, 1, True, 4.4)
def testExample2(self):
inputs = [[2.2, 4.4, 1.1], [5.5, 3.3, 6.6]]
self._validateNthElement(inputs, dtypes.float64, 2, False, [4.4, 6.6])
self._validateNthElement(inputs, dtypes.float64, 2, True, [1.1, 3.3])
def testExample3(self):
inputs = [[[2, 4, 1], [5, -3, 6]],
[[7, 9, -8], [9, 0, 4]]]
self._validateNthElement(inputs, dtypes.int32, 0, False,
[[1, -3], [-8, 0]])
self._validateNthElement(inputs, dtypes.int64, 0, True,
[[4, 6], [9, 9]])
def _testFloatLargeInput(self, input_shape):
inputs = np.random.random_sample(input_shape)
n = np.random.randint(input_shape[-1])
sort_inputs = np.sort(inputs)
expected_values = sort_inputs[..., n]
self._validateNthElement(
inputs, dtypes.float32, n, False, expected_values)
expected_values = sort_inputs[..., ::-1][..., n]
self._validateNthElement(
inputs, dtypes.float64, n, True, expected_values)
def _testIntLargeInput(self, input_shape):
inputs = np.random.randint(-1e3, 1e3, input_shape)
n = np.random.randint(input_shape[-1])
sort_inputs = np.sort(inputs)
expected_values = sort_inputs[..., n]
self._validateNthElement(
inputs, dtypes.int32, n, False, expected_values)
expected_values = sort_inputs[..., ::-1][..., n]
self._validateNthElement(
inputs, dtypes.int64, n, True, expected_values)
def _testLargeInput(self, input_shape):
self._testFloatLargeInput(input_shape)
self._testIntLargeInput(input_shape)
def testLargeInput(self):
self._testLargeInput([1])
self._testLargeInput([10])
self._testLargeInput([5, 10])
self._testLargeInput([50, 100])
self._testLargeInput([50, 10000])
self._testLargeInput([50, 10, 100])
self._testLargeInput([50, 10, 10, 100])
def _testEnumerateN(self, input_shape):
inputs = np.random.random_sample(input_shape)
sort_inputs = np.sort(inputs)
for n in range(input_shape[-1]):
expected_values = sort_inputs[..., n]
self._validateNthElement(
inputs, dtypes.float32, n, False, expected_values)
expected_values = sort_inputs[..., ::-1][..., n]
self._validateNthElement(
inputs, dtypes.float64, n, True, expected_values)
def testEnumerateN(self):
self._testEnumerateN([1])
self._testEnumerateN([10])
self._testEnumerateN([10, 10])
self._testEnumerateN([10, 10, 10])
self._testEnumerateN([10, 10, 10, 10])
def testInvalidInput(self):
with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError),
"at least rank 1 but is rank 0"):
nn_ops.nth_element(5, 0)
# Test with placeholders
with ops.Graph().as_default():
with self.session(use_gpu=False):
v = array_ops.placeholder(dtype=dtypes.int32)
with self.assertRaisesOpError("at least rank 1 but is rank 0"):
nn_ops.nth_element(v, 0).eval(feed_dict={v: 5})
def testInvalidN(self):
with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError),
"non-negative but is -1"):
nn_ops.nth_element([5], -1)
with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError),
"scalar but has rank 1"):
nn_ops.nth_element([5, 6, 3], [1])
# Test with placeholders
with ops.Graph().as_default():
with self.session(use_gpu=False):
n = array_ops.placeholder(dtypes.int32)
values = nn_ops.nth_element([5], n)
with self.assertRaisesOpError("non-negative but is -1"):
values.eval(feed_dict={n: -1})
def testNTooLarge(self):
inputs = [[0.1, 0.2], [0.3, 0.4]]
with self.assertRaisesRegex((ValueError, errors.InvalidArgumentError),
"must have last dimension > n = 2"):
nn_ops.nth_element(inputs, 2)
# Test with placeholders
with ops.Graph().as_default():
with self.session(use_gpu=False):
n = array_ops.placeholder(dtypes.int32)
values = nn_ops.nth_element(inputs, n)
with self.assertRaisesOpError("must have last dimension > n = 2"):
values.eval(feed_dict={n: 2})
def testGradients(self):
x = [
[2., -1., 1000., 3., 1000.],
[1., 5., 2., 4., 3.],
[2., 2., 2., 2., 2.],
]
grad_ys = [[-1., 2., 5.]]
result = [
[0, 0, -0.5, 0, -0.5],
[0, 0, 0, 2, 0],
[1, 1, 1, 1, 1],
]
if context.executing_eagerly():
inputs = ops.convert_to_tensor(x)
with backprop.GradientTape() as tape:
tape.watch(inputs)
values = nn_ops.nth_element(inputs, 3)
grad = tape.gradient(values, inputs, ops.convert_to_tensor(grad_ys))
self.assertAllClose(grad[0], result)
# Test with tf.gradients
with ops.Graph().as_default():
with self.session(use_gpu=False) as sess:
inputs = array_ops.placeholder(dtypes.float32, shape=[3, 5])
values = nn_ops.nth_element(inputs, 3)
grad = sess.run(
gradients_impl.gradients(values, inputs, grad_ys=grad_ys),
feed_dict={inputs: x})
self.assertAllClose(grad[0], result)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/pool_test.py
+386
View File
@@ -0,0 +1,386 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for unified pooling functionality in tensorflow.ops.nn."""
import math
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
def pool_direct_single_axis(
input, # pylint: disable=redefined-builtin
axis,
window_size,
pooling_type,
padding,
dilation_rate,
stride):
"""Numpy implementation of pooling along a single axis.
This is intended for testing only, and therefore isn't particularly efficient.
See pool_direct below for the meaning of the arguments.
Args:
input: numpy array.
axis: axis along which to perform pooling.
window_size: int >= 1. Size of pooling window within axis.
pooling_type: either "MAX" or "AVG".
padding: either "SAME" or "VALID".
dilation_rate: int >= 1. Dilation factor for window, i.e. stride at which
to sample input.
stride: int >= 1. Stride at which to generate output.
Returns:
pooling output array of rank N+2.
Raises:
ValueError: if arguments are invalid.
"""
effective_window_size = (window_size - 1) * dilation_rate + 1
input_size = input.shape[axis]
if padding == "SAME":
output_size = int(math.ceil(input_size / stride))
total_padding_amount = max(0, (output_size - 1) * stride +
effective_window_size - input_size)
before_padding = total_padding_amount // 2
elif padding == "VALID":
output_size = int(
math.ceil((input_size - effective_window_size + 1) / stride))
before_padding = 0
else:
raise ValueError("Unsupported padding type: %r" % (padding,))
output_shape = input.shape[:axis] + (output_size,) + input.shape[axis + 1:]
output = np.zeros(output_shape, input.dtype)
initial_dim_selector = tuple(np.s_[:] for _ in range(axis))
if pooling_type == "MAX":
pooling_func = np.max
elif pooling_type == "AVG":
pooling_func = np.mean
else:
raise ValueError("Unsupported pooling type: %r" % (pooling_type,))
for output_pos in range(output_size):
input_start_pos = output_pos * stride - before_padding
input_end_pos = min(input_start_pos + effective_window_size, input_size)
if input_start_pos < 0:
input_start_pos += dilation_rate
input_slice = np.s_[input_start_pos:input_end_pos:dilation_rate]
output[initial_dim_selector + (output_pos,)] = pooling_func(
input[initial_dim_selector + (input_slice,)], axis=axis)
return output
def pool_direct(
input, # pylint: disable=redefined-builtin
window_shape,
pooling_type,
padding, # pylint: disable=redefined-builtin
dilation_rate,
strides,
data_format=None):
"""Numpy implementation of pooling.
This is intended for testing only, and therefore isn't particularly efficient.
See tensorflow.nn.pool.
Args:
input: numpy array of rank N+2.
window_shape: Sequence of N ints >= 1.
pooling_type: either "MAX" or "AVG".
padding: either "SAME" or "VALID".
dilation_rate: Sequence of N ints >= 1.
strides: Sequence of N ints >= 1.
data_format: If specified and starts with "NC", indicates that second
dimension, rather than the last dimension, specifies the channel.
Returns:
pooling output array of rank N+2.
Raises:
ValueError: if arguments are invalid.
"""
if data_format is None or not data_format.startswith("NC"):
spatial_start_dim = 1
else:
spatial_start_dim = 2
output = input
for i in range(len(window_shape)):
output = pool_direct_single_axis(
input=output,
axis=i + spatial_start_dim,
window_size=window_shape[i],
pooling_type=pooling_type,
padding=padding,
dilation_rate=dilation_rate[i],
stride=strides[i])
return output
class PoolingTest(test.TestCase):
def _test(self, input_shape, **kwargs):
# Use negative numbers to make sure there isn't any zero padding getting
# used.
x = -np.arange(
np.prod(input_shape), dtype=np.float32).reshape(input_shape) - 1
y1 = pool_direct(input=x, **kwargs)
y2 = nn_ops.pool(input=x, **kwargs)
self.assertAllClose(y1, self.evaluate(y2), rtol=1e-2, atol=1e-2)
def testPoolSimple(self):
with self.session(use_gpu=test.is_gpu_available()):
for padding in ["SAME", "VALID"]:
for pooling_type in ["MAX", "AVG"]:
self._test(
input_shape=[1, 1, 10, 1],
window_shape=[1, 3],
padding=padding,
pooling_type=pooling_type,
dilation_rate=[1, 1],
strides=[1, 2])
def testPool1D(self):
with self.session(use_gpu=test.is_gpu_available()):
for padding in ["SAME", "VALID"]:
for pooling_type in ["MAX", "AVG"]:
for input_shape in [[2, 9, 2], [2, 10, 2]]:
for window_shape in [[1], [2], [3]]:
if padding != "SAME":
for dilation_rate in [[1], [2], [3]]:
self._test(
input_shape=input_shape,
window_shape=window_shape,
padding=padding,
pooling_type=pooling_type,
dilation_rate=dilation_rate,
strides=[1])
for strides in [[1], [2], [3]]:
if np.any(np.array(strides) > window_shape):
continue
self._test(
input_shape=input_shape,
window_shape=window_shape,
padding=padding,
pooling_type=pooling_type,
dilation_rate=[1],
strides=strides)
def testPool2D(self):
with self.session(use_gpu=test.is_gpu_available()):
for padding in ["SAME", "VALID"]:
for pooling_type in ["MAX", "AVG"]:
for input_shape in [[2, 9, 10, 2], [2, 10, 9, 2]]:
for window_shape in [[1, 1], [2, 1], [2, 3]]:
if padding != "SAME":
for dilation_rate in [[1, 1], [2, 1], [1, 2], [2, 3]]:
self._test(
input_shape=input_shape,
window_shape=window_shape,
padding=padding,
pooling_type=pooling_type,
dilation_rate=dilation_rate,
strides=[1, 1])
for strides in [[1, 1], [2, 1], [1, 2], [2, 3]]:
if np.any(np.array(strides) > window_shape):
continue
self._test(
input_shape=input_shape,
window_shape=window_shape,
padding=padding,
pooling_type=pooling_type,
dilation_rate=[1, 1],
strides=strides)
def testPool3D(self):
with self.session(use_gpu=test.is_gpu_available()):
for padding in ["SAME", "VALID"]:
for pooling_type in ["MAX", "AVG"]:
for input_shape in [[2, 9, 10, 11, 2], [2, 10, 9, 11, 2]]:
for window_shape in [[1, 1, 1], [2, 1, 2], [2, 3, 2]]:
if padding != "SAME":
for dilation_rate in [[1, 1, 1], [2, 1, 2], [1, 2, 2],
[2, 3, 3]]:
self._test(
input_shape=input_shape,
window_shape=window_shape,
padding=padding,
pooling_type=pooling_type,
dilation_rate=dilation_rate,
strides=[1, 1, 1])
for strides in [[1, 1, 1], [2, 1, 2], [1, 2, 2], [2, 3, 3]]:
if np.any(np.array(strides) > window_shape):
continue
self._test(
input_shape=input_shape,
window_shape=window_shape,
padding=padding,
pooling_type=pooling_type,
dilation_rate=[1, 1, 1],
strides=strides)
def testPoolNC(self):
if test.is_gpu_available(cuda_only=True):
# "NC*" format is currently only supported on CUDA.
with self.session():
for padding in ["SAME", "VALID"]:
self._test(
input_shape=[2, 2, 9],
window_shape=[2],
padding=padding,
pooling_type="MAX",
strides=[1],
dilation_rate=[1],
data_format="NCW")
self._test(
input_shape=[2, 2, 9],
window_shape=[2],
padding=padding,
pooling_type="MAX",
strides=[2],
dilation_rate=[1],
data_format="NCW")
self._test(
input_shape=[2, 2, 7, 9],
window_shape=[2, 2],
padding=padding,
pooling_type="MAX",
strides=[1, 2],
dilation_rate=[1, 1],
data_format="NCHW")
self._test(
input_shape=[2, 2, 7, 5, 3],
window_shape=[2, 2, 2],
padding=padding,
pooling_type="MAX",
strides=[1, 2, 1],
dilation_rate=[1, 1, 1],
data_format="NCDHW")
self._test(
input_shape=[2, 2, 7, 9],
window_shape=[2, 2],
padding="VALID",
pooling_type="MAX",
strides=[1, 1],
dilation_rate=[2, 2],
data_format="NCHW")
def _test_gradient(self, input_shape, **kwargs):
x_val = -np.arange(
np.prod(input_shape), dtype=np.float32).reshape(input_shape) - 1
x = constant_op.constant(x_val, name="x", dtype=dtypes.float32)
output = nn_ops.pool(input=x, **kwargs)
y_shape = output.get_shape().as_list()
err = gradient_checker.compute_gradient_error([x], [input_shape],
output,
y_shape,
x_init_value=[x_val])
err_tolerance = 1e-2
self.assertLess(err, err_tolerance)
@test_util.run_deprecated_v1
def testGradient1D(self):
with self.session(use_gpu=test.is_gpu_available()):
for padding in ["SAME", "VALID"]:
for pooling_type in ["AVG", "MAX"]:
for input_shape in [[2, 5, 2], [1, 4, 1]]:
for window_shape in [[1], [2]]:
if padding != "SAME":
for dilation_rate in [[1], [2]]:
self._test_gradient(
input_shape=input_shape,
window_shape=window_shape,
padding=padding,
pooling_type=pooling_type,
dilation_rate=dilation_rate,
strides=[1])
for strides in [[1], [2]]:
if np.any(np.array(strides) > window_shape):
continue
self._test(
input_shape=input_shape,
window_shape=window_shape,
padding=padding,
pooling_type=pooling_type,
dilation_rate=[1],
strides=strides)
@test_util.run_deprecated_v1
def testGradient2D(self):
with self.session(use_gpu=test.is_gpu_available()):
for padding in ["SAME", "VALID"]:
for pooling_type in ["AVG", "MAX"]:
for input_shape in [[2, 4, 5, 2], [1, 5, 4, 1]]:
for window_shape in [[1, 1], [2, 1], [2, 2]]:
if padding != "SAME":
for dilation_rate in [[1, 1], [2, 1], [2, 2]]:
self._test_gradient(
input_shape=input_shape,
window_shape=window_shape,
padding=padding,
pooling_type=pooling_type,
dilation_rate=dilation_rate,
strides=[1, 1])
for strides in [[1, 1], [2, 1], [1, 2], [2, 2]]:
if np.any(np.array(strides) > window_shape):
continue
self._test(
input_shape=input_shape,
window_shape=window_shape,
padding=padding,
pooling_type=pooling_type,
dilation_rate=[1, 1],
strides=strides)
@test_util.run_deprecated_v1
def testGradient3D(self):
with self.session(use_gpu=test.is_gpu_available()):
for padding in ["SAME", "VALID"]:
for pooling_type in ["AVG", "MAX"]:
for input_shape in [[1, 3, 5, 4, 1], [1, 5, 4, 3, 1]]:
for window_shape in [[1, 1, 1], [2, 1, 2], [2, 2, 2]]:
if padding != "SAME":
for dilation_rate in [[1, 1, 1], [2, 1, 2], [2, 2, 2]]:
self._test_gradient(
input_shape=input_shape,
window_shape=window_shape,
padding=padding,
pooling_type=pooling_type,
dilation_rate=dilation_rate,
strides=[1, 1, 1])
for strides in [[1, 1, 1], [2, 1, 2], [2, 2, 2]]:
if np.any(np.array(strides) > window_shape):
continue
self._test(
input_shape=input_shape,
window_shape=window_shape,
padding=padding,
pooling_type=pooling_type,
dilation_rate=[1, 1, 1],
strides=strides)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/pooling_ops_3d_test.py
+672
View File
@@ -0,0 +1,672 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Functional tests for 3d pooling operations."""
import numpy as np
from tensorflow.python.eager import backprop
from tensorflow.python.eager import context
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_nn_ops
from tensorflow.python.ops import gradient_checker_v2
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
def GetTestConfigs():
"""Get all the valid tests configs to run.
Returns:
all the valid test configs as tuples of data_format and use_gpu.
"""
test_configs = [("NDHWC", False), ("NDHWC", True)]
if test.is_gpu_available(cuda_only=True):
# "NCHW" format is currently supported exclusively on CUDA GPUs.
test_configs += [("NCDHW", True)]
return test_configs
# TODO(mjanusz): Add microbenchmarks for 3d pooling.
@test_util.with_eager_op_as_function
class PoolingTest(test.TestCase):
def _VerifyOneTest(self, pool_func, input_sizes, window, strides, padding,
data_format, data_type, expected, use_gpu):
"""Verifies the output values of the pooling function.
Args:
pool_func: Function to be called: co.MaxPool, co.AvgPool.
input_sizes: Input tensor dimensions.
window: Tuple of kernel dims: planes, rows, cols.
strides: Tuple of strides for dims: planes, rows, cols.
padding: Padding type.
data_format: The data format we use to run the pooling operation.
data_type: The data type to use to run the pooling operation.
expected: An array containing the expected operation outputs.
use_gpu: Whether to run ops on GPU.
"""
total_size = 1
for s in input_sizes:
total_size *= s
# Initializes the input tensor with array containing incrementing
# numbers from 1.
x = [f * 1.0 for f in range(1, total_size + 1)]
if data_type == dtypes.bfloat16:
x = [f * 0.1 for f in x]
expected = [f * 0.1 for f in expected]
with self.cached_session(use_gpu=use_gpu):
t = constant_op.constant(x, shape=input_sizes, dtype=data_type)
window = [1] + list(window) + [1]
strides = [1] + list(strides) + [1]
if data_format == "NCDHW":
t = test_util.NHWCToNCHW(t)
window = test_util.NHWCToNCHW(window)
strides = test_util.NHWCToNCHW(strides)
t = pool_func(
t,
ksize=window,
strides=strides,
padding=padding,
data_format=data_format)
if data_format == "NCDHW":
t = test_util.NCHWToNHWC(t)
vals = self.evaluate(t)
# Verifies values.
actual = vals.flatten()
rtol = atol = 1e-6
if data_type == dtypes.bfloat16:
rtol = atol = 2e-2
self.assertAllClose(expected, actual, rtol=rtol, atol=atol)
def _VerifyValues(self, pool_func, input_sizes, window, strides,
padding, expected):
for data_format, use_gpu in GetTestConfigs():
self._VerifyOneTest(pool_func, input_sizes, window, strides, padding,
data_format, dtypes.float32, expected, use_gpu)
# Don't test bfloat16 on GPU if there is no GPU available.
if (not use_gpu) or test_util.is_gpu_available(cuda_only=True):
self._VerifyOneTest(pool_func, input_sizes, window, strides, padding,
data_format, dtypes.bfloat16, expected, use_gpu)
def testAvgPool3dValidPadding(self):
expected_output = [20.5, 21.5, 22.5]
self._VerifyValues(
nn_ops.avg_pool3d,
input_sizes=[1, 3, 3, 3, 3],
window=(2, 2, 2),
strides=(2, 2, 2),
padding="VALID",
expected=expected_output)
def testAvgPool3dSamePadding(self):
expected_output = [20.5, 21.5, 22.5, 26.5, 27.5, 28.5]
self._VerifyValues(
nn_ops.avg_pool3d,
input_sizes=[1, 2, 2, 4, 3],
window=(2, 2, 2),
strides=(2, 2, 2),
padding="SAME",
expected=expected_output)
def testAvgPool3dSamePaddingDifferentStrides(self):
expected_output = [1.5, 4.5, 7.5, 17.5, 20.5, 23.5, 33.5, 36.5, 39.5]
self._VerifyValues(
nn_ops.avg_pool3d,
input_sizes=[1, 5, 8, 1, 1],
window=(1, 2, 3),
strides=(2, 3, 1),
padding="SAME",
expected=expected_output)
def testAvgPool3dGrad(self):
with self.assertRaises(
(errors.ResourceExhaustedError, errors.InvalidArgumentError)):
for dtype in [dtypes.float32, dtypes.bfloat16]:
with self.cached_session():
orig_input_shape = constant_op.constant(
1879048192, shape=[5], dtype=dtypes.int32
)
grad = constant_op.constant(1, shape=[1, 3, 2, 4, 2], dtype=dtype)
t = gen_nn_ops.AvgPool3DGrad(
orig_input_shape=orig_input_shape,
grad=grad,
ksize=[1, 1, 1, 1, 1],
strides=[1, 1, 1, 1, 1],
padding="SAME",
data_format="NDHWC",
)
self.evaluate(t)
def testAvgPool3dGradEmptyInput(self):
for data_format, use_gpu in GetTestConfigs():
with self.cached_session(use_gpu=use_gpu):
orig_input_shape = constant_op.constant([5, 6, 7, 0, 8],
dtype=dtypes.int32)
grad = constant_op.constant(
1, shape=[5, 6, 7, 0, 8], dtype=dtypes.float32)
t = gen_nn_ops.AvgPool3DGrad(
orig_input_shape=orig_input_shape,
grad=grad,
ksize=[1, 1, 1, 1, 1],
strides=[1, 1, 1, 1, 1],
padding="SAME",
data_format=data_format)
self.evaluate(t)
def testAvgPool3dGradInvalidKsize(self):
for data_format, use_gpu in GetTestConfigs():
with self.cached_session(use_gpu=use_gpu):
orig_input_shape = constant_op.constant(
[1, 1, 1, 1, 1], dtype=dtypes.int32
)
grad = [[[[[1.0]]]]]
with self.assertRaises(errors.InvalidArgumentError):
t = gen_nn_ops.AvgPool3DGrad(
orig_input_shape=orig_input_shape,
grad=grad,
ksize=[1, -3, 1, 1, 1],
strides=[1, 1, 1, 1, 1],
padding="SAME",
data_format=data_format,
)
self.evaluate(t)
def testMaxPool3dValidPadding(self):
expected_output = [40.0, 41.0, 42.0]
self._VerifyValues(
nn_ops.max_pool3d,
input_sizes=[1, 3, 3, 3, 3],
window=(2, 2, 2),
strides=(2, 2, 2),
padding="VALID",
expected=expected_output)
def testMaxPool3dSamePadding(self):
expected_output = [31., 32., 33., 34., 35., 36.]
self._VerifyValues(
nn_ops.max_pool3d,
input_sizes=[1, 2, 2, 3, 3],
window=(2, 2, 2),
strides=(2, 2, 2),
padding="SAME",
expected=expected_output)
def testMaxPool3dSamePaddingDifferentStrides(self):
expected_output = [2., 5., 8., 18., 21., 24., 34., 37., 40.]
self._VerifyValues(
nn_ops.max_pool3d,
input_sizes=[1, 5, 8, 1, 1],
window=(1, 2, 3),
strides=(2, 3, 1),
padding="SAME",
expected=expected_output)
# Test pooling on a larger input, with different stride and kernel
# size for the 'z' dimension.
# Simulate max pooling in numpy to get the expected output.
input_data = np.arange(1, 5 * 27 * 27 * 64 + 1).reshape((5, 27, 27, 64))
input_data = np.pad(input_data, [[0, 0], [0, 1], [0, 1], [0, 0]],
mode="constant")
expected_output = input_data[:, 1::2, 1::2, :]
expected_output[:, -1, :, :] = input_data[:, -2, 1::2, :]
expected_output[:, :, -1, :] = input_data[:, 1::2, -2, :]
expected_output[:, -1, -1, :] = input_data[:, -2, -2, :]
self._VerifyValues(
nn_ops.max_pool3d,
input_sizes=[1, 5, 27, 27, 64],
window=(1, 2, 2),
strides=(1, 2, 2),
padding="SAME",
expected=expected_output.flatten())
def testKernelSmallerThanStride(self):
self._VerifyValues(
nn_ops.max_pool3d,
input_sizes=[1, 3, 3, 3, 1],
window=[1, 1, 1],
strides=[2, 2, 2],
padding="SAME",
expected=[1, 3, 7, 9, 19, 21, 25, 27])
self._VerifyValues(
nn_ops.max_pool3d,
input_sizes=[1, 7, 7, 7, 1],
window=[2, 2, 2],
strides=[3, 3, 3],
padding="VALID",
expected=[58, 61, 79, 82, 205, 208, 226, 229])
self._VerifyValues(
nn_ops.avg_pool3d,
input_sizes=[1, 3, 3, 3, 1],
window=[1, 1, 1],
strides=[2, 2, 2],
padding="SAME",
expected=[1, 3, 7, 9, 19, 21, 25, 27])
self._VerifyValues(
nn_ops.avg_pool3d,
input_sizes=[1, 7, 7, 7, 1],
window=[2, 2, 2],
strides=[3, 3, 3],
padding="VALID",
expected=[29.5, 32.5, 50.5, 53.5, 176.5, 179.5, 197.5, 200.5])
def testMaxPool3DEmptyTensorOutputShape(self):
"""Verifies the output shape of the max pooling function when tensor is empty.
Args: none
"""
input_sizes = [0, 112, 112, 112, 64]
input_data = 1.
input_tensor = constant_op.constant(
input_data, shape=input_sizes, name="input")
max_pool_3d = nn_ops.max_pool3d(
input_tensor,
ksize=[2, 2, 2],
strides=[2, 2, 2],
padding="VALID",
data_format="NDHWC",
name="max_pool_3d")
values = self.evaluate(max_pool_3d)
self.assertEqual(values.shape, (0, 56, 56, 56, 64))
# TODO(penporn): Determine if we will allow input_sizes[3] < ksize[3].
def DISABLED_testAvgPool3dEmptyOutTensor(self):
input_sizes = [30, 19, 4, 19, 17]
input_data = 1.0
input_tensor = constant_op.constant(
input_data, shape=input_sizes, name="input")
avg_pool_3d = nn_ops.avg_pool3d(
input_tensor,
ksize=(1, 13, 3, 20, 1),
strides=(1, 14, 4, 1, 1),
padding="VALID",
data_format="NDHWC",
name="avg_pool_3d")
values = self.evaluate(avg_pool_3d)
self.assertEqual(values.shape, (30, 1, 1, 0, 17))
def _getJacobians(self,
pool_func,
input_sizes,
output_sizes,
window,
strides,
padding,
data_format,
use_gpu,
dtype=np.float32):
with self.cached_session(use_gpu=use_gpu):
x = np.arange(np.prod(input_sizes)).reshape(input_sizes).astype(dtype)
input_tensor = constant_op.constant(x, shape=input_sizes)
ksize = [1, window[0], window[1], window[2], 1]
strides = [1, strides[0], strides[1], strides[2], 1]
if data_format == "NCDHW":
ksize = test_util.NHWCToNCHW(ksize)
strides = test_util.NHWCToNCHW(strides)
input_tensor = test_util.NHWCToNCHW(input_tensor)
output_sizes = test_util.NHWCToNCHW(output_sizes)
def func(in_tensor):
return pool_func(
in_tensor,
ksize=ksize,
strides=strides,
padding=padding,
data_format=data_format)
input_jacob_a, input_jacob_n = gradient_checker_v2.compute_gradient(
func, [input_tensor])
def pool_grad_function(upstream_gradients):
with backprop.GradientTape() as tape:
tape.watch(input_tensor)
pool_output = pool_func(
input_tensor,
ksize=ksize,
strides=strides,
padding=padding,
data_format=data_format)
gradient_injector_output = pool_output * upstream_gradients
return tape.gradient(gradient_injector_output, input_tensor)
upstream_tensor = constant_op.constant(
2 * np.random.rand(*output_sizes) - 1, dtype=dtype)
grad_jacob_a, grad_jacob_n = gradient_checker_v2.compute_gradient(
pool_grad_function, [upstream_tensor])
return ((input_jacob_a, grad_jacob_a), (input_jacob_n, grad_jacob_n))
def _ConstructAndTestGradientForConfig(self, pool_func, input_sizes,
output_sizes, window, strides, padding,
data_format, data_type, use_gpu):
"""Verifies the gradients of a pooling function.
Args:
pool_func: Function to be called, co.MaxPool, co.AvgPool,
or the Lua version.
input_sizes: Input tensor dimensions.
output_sizes: Output tensor dimensions.
window: Tuple of kernel dims: planes, rows, cols.
strides: Tuple of strides for dims: planes, rows, cols.
padding: Padding type.
data_format: Data format string.
data_type: The data type to use to run the pooling operation.
use_gpu: Whether to run on GPU.
"""
jacob_a, jacob_n = self._getJacobians(
pool_func,
input_sizes,
output_sizes,
window,
strides,
padding,
data_format,
use_gpu,
dtype=data_type.as_numpy_dtype)
if data_type == dtypes.bfloat16:
# Compare bf16 analytical gradients to fp32 numerical gradients.
_, jacob_n = self._getJacobians(
pool_func,
input_sizes,
output_sizes,
window,
strides,
padding,
data_format,
use_gpu,
dtype=np.float32)
input_jacob_a, grad_jacob_a = jacob_a
input_jacob_n, grad_jacob_n = jacob_n
self.assertAllClose(input_jacob_a, input_jacob_n, rtol=1e-3, atol=1e-3)
self.assertAllClose(grad_jacob_a, grad_jacob_n, rtol=1e-3, atol=1e-3)
def _ConstructAndTestGradient(self,
pool_func,
**kwargs):
"""Runs _ConstructAndTestGradientForConfig for all tests configurations."""
for data_format, use_gpu in GetTestConfigs():
self._ConstructAndTestGradientForConfig(
pool_func,
data_format=data_format,
data_type=dtypes.float32,
use_gpu=use_gpu,
**kwargs)
if use_gpu and test_util.is_gpu_available(cuda_only=True):
self._ConstructAndTestGradientForConfig(
pool_func,
data_format=data_format,
data_type=dtypes.bfloat16,
use_gpu=use_gpu,
**kwargs)
@test_util.run_deprecated_v1
def testMaxPoolGradValidPadding1_1_3d(self):
self._ConstructAndTestGradient(
nn_ops.max_pool3d,
input_sizes=[1, 3, 3, 3, 1],
output_sizes=[1, 3, 3, 3, 1],
window=(1, 1, 1),
strides=(1, 1, 1),
padding="VALID")
@test_util.run_deprecated_v1
def testMaxPoolGradValidPadding2_1_6_3d(self):
self._ConstructAndTestGradient(
nn_ops.max_pool3d,
input_sizes=[1, 2, 3, 4, 2],
output_sizes=[1, 1, 2, 3, 2],
window=(2, 2, 2),
strides=(1, 1, 1),
padding="VALID")
@test_util.run_deprecated_v1
def testMaxPoolGradValidPadding2_1_7_3d(self):
self._ConstructAndTestGradient(
nn_ops.max_pool3d,
input_sizes=[1, 3, 2, 7, 1],
output_sizes=[1, 2, 1, 6, 1],
window=(2, 2, 2),
strides=(1, 1, 1),
padding="VALID")
@test_util.run_deprecated_v1
def testMaxPoolGradValidPadding1_2_3d(self):
self._ConstructAndTestGradient(
nn_ops.max_pool3d,
input_sizes=[1, 3, 3, 3, 1],
output_sizes=[1, 2, 2, 2, 1],
window=(1, 1, 1),
strides=(2, 2, 2),
padding="VALID")
@test_util.run_deprecated_v1
def testMaxPoolGradValidPadding2_2_3d(self):
self._ConstructAndTestGradient(
nn_ops.max_pool3d,
input_sizes=[2, 2, 2, 2, 1],
output_sizes=[2, 1, 1, 1, 1],
window=(2, 2, 2),
strides=(2, 2, 2),
padding="VALID")
@test_util.run_deprecated_v1
def testMaxPoolGradSamePadding1_1_3d(self):
self._ConstructAndTestGradient(
nn_ops.max_pool3d,
input_sizes=[1, 3, 2, 4, 1],
output_sizes=[1, 3, 2, 4, 1],
window=(1, 1, 1),
strides=(1, 1, 1),
padding="SAME")
@test_util.run_deprecated_v1
def testMaxPoolGradSamePadding1_2_3d(self):
self._ConstructAndTestGradient(
nn_ops.max_pool3d,
input_sizes=[1, 3, 2, 4, 1],
output_sizes=[1, 2, 1, 2, 1],
window=(1, 1, 1),
strides=(2, 2, 2),
padding="SAME")
@test_util.run_deprecated_v1
def testMaxPoolGradSamePadding2_1_3d(self):
self._ConstructAndTestGradient(
nn_ops.max_pool3d,
input_sizes=[1, 3, 2, 4, 1],
output_sizes=[1, 3, 2, 4, 1],
window=(2, 2, 2),
strides=(1, 1, 1),
padding="SAME")
@test_util.run_deprecated_v1
def testMaxPoolGradSamePadding2_2_3d(self):
self._ConstructAndTestGradient(
nn_ops.max_pool3d,
input_sizes=[1, 5, 2, 4, 2],
output_sizes=[1, 3, 1, 2, 2],
window=(2, 2, 2),
strides=(2, 2, 2),
padding="SAME")
@test_util.run_deprecated_v1
def testMaxPoolGradSamePadding3_1_3d(self):
self._ConstructAndTestGradient(
nn_ops.max_pool3d,
input_sizes=[1, 3, 4, 2, 1],
output_sizes=[1, 3, 4, 2, 1],
window=(3, 3, 3),
strides=(1, 1, 1),
padding="SAME")
@test_util.run_deprecated_v1
def testAvgPoolGradValidPadding1_1_3d(self):
self._ConstructAndTestGradient(
nn_ops.avg_pool3d,
input_sizes=[1, 3, 3, 3, 1],
output_sizes=[1, 3, 3, 3, 1],
window=(1, 1, 1),
strides=(1, 1, 1),
padding="VALID")
@test_util.run_deprecated_v1
def testAvgPoolGradValidPadding1_2_3d(self):
self._ConstructAndTestGradient(
nn_ops.avg_pool3d,
input_sizes=[1, 3, 3, 3, 1],
output_sizes=[1, 2, 2, 2, 1],
window=(1, 1, 1),
strides=(2, 2, 2),
padding="VALID")
@test_util.run_deprecated_v1
def testAvgPoolGradValidPadding2_1_3d(self):
self._ConstructAndTestGradient(
nn_ops.avg_pool3d,
input_sizes=[1, 3, 3, 3, 2],
output_sizes=[1, 2, 2, 2, 2],
window=(2, 2, 2),
strides=(1, 1, 1),
padding="VALID")
@test_util.run_deprecated_v1
def testAvgPoolGradValidPadding2_2_3d(self):
self._ConstructAndTestGradient(
nn_ops.avg_pool3d,
input_sizes=[2, 2, 2, 2, 2],
output_sizes=[2, 1, 1, 1, 2],
window=(2, 2, 2),
strides=(2, 2, 2),
padding="VALID")
@test_util.run_deprecated_v1
def testAvgPoolGradSamePadding1_1_3d(self):
self._ConstructAndTestGradient(
nn_ops.avg_pool3d,
input_sizes=[1, 3, 2, 4, 2],
output_sizes=[1, 3, 2, 4, 2],
window=(1, 1, 1),
strides=(1, 1, 1),
padding="SAME")
@test_util.run_deprecated_v1
def testAvgPoolGradSamePadding1_2_3d(self):
self._ConstructAndTestGradient(
nn_ops.avg_pool3d,
input_sizes=[1, 3, 2, 4, 2],
output_sizes=[1, 2, 1, 2, 2],
window=(1, 1, 1),
strides=(2, 2, 2),
padding="SAME")
@test_util.run_deprecated_v1
def testAvgPoolGradSamePadding2_1_3d(self):
self._ConstructAndTestGradient(
nn_ops.avg_pool3d,
input_sizes=[1, 2, 2, 2, 1],
output_sizes=[1, 2, 2, 2, 1],
window=(2, 2, 2),
strides=(1, 1, 1),
padding="SAME")
@test_util.run_deprecated_v1
def testAvgPoolGradSamePadding2_2_3d(self):
self._ConstructAndTestGradient(
nn_ops.avg_pool3d,
input_sizes=[1, 5, 2, 4, 1],
output_sizes=[1, 3, 1, 2, 1],
window=(2, 2, 2),
strides=(2, 2, 2),
padding="SAME")
@test_util.run_deprecated_v1
def testAvgPoolGradSamePadding3_1_3d(self):
self._ConstructAndTestGradient(
nn_ops.avg_pool3d,
input_sizes=[1, 3, 6, 2, 1],
output_sizes=[1, 3, 6, 2, 1],
window=(3, 3, 3),
strides=(1, 1, 1),
padding="SAME")
def testMaxPool3DZeroPoolSize(self):
# Test case for GitHub issue 51936.
for f in [nn_ops.max_pool3d, nn_ops.avg_pool3d]:
with self.session():
with self.assertRaises((errors.InvalidArgumentError, ValueError)):
input_sizes = [3, 4, 10, 11, 12]
input_data = 1.
input_tensor = constant_op.constant(
input_data, shape=input_sizes, name="input")
pool_3d = f(input_tensor, ksize=[2, 2, 0], strides=1, padding="VALID")
self.evaluate(pool_3d)
@test_util.disable_xla("b/205634417") # XLA does not raise these errors.
def testMaxPoolGradEagerShapeErrors(self):
with context.eager_mode():
orig_in = array_ops.ones((1, 1, 1, 1, 1))
# Test invalid orig_out shape
orig_out = array_ops.ones((1, 1, 1, 1, 2))
grad = array_ops.ones((1, 1, 1, 1, 1))
with self.assertRaisesRegex(
errors_impl.InvalidArgumentError,
r"Expected orig_output shape to be \[1,1,1,1,1\], but got "
r"\[1,1,1,1,2\]"):
gen_nn_ops.max_pool3d_grad(
orig_in, orig_out, grad, ksize=[1, 1, 1, 1, 1],
strides=[1, 1, 1, 1, 1], padding="VALID")
with self.assertRaisesRegex(
errors_impl.InvalidArgumentError,
r"Expected orig_output shape to be \[1,1,1,1,1\], but got "
r"\[1,1,1,1,2\]"):
gen_nn_ops.max_pool3d_grad_grad(
orig_in, orig_out, grad, ksize=[1, 1, 1, 1, 1],
strides=[1, 1, 1, 1, 1], padding="VALID")
# Test invalid grad shape
orig_out = array_ops.ones((1, 1, 1, 1, 1))
grad = array_ops.ones((1, 1, 1, 1, 2))
with self.assertRaisesRegex(
errors_impl.InvalidArgumentError,
r"Expected grad shape to be \[1,1,1,1,1\], but got \[1,1,1,1,2\]"):
gen_nn_ops.max_pool3d_grad(
orig_in, orig_out, grad, ksize=[1, 1, 1, 1, 1],
strides=[1, 1, 1, 1, 1], padding="VALID")
with self.assertRaisesRegex(
errors_impl.InvalidArgumentError,
r"Expected grad shape to be \[1,1,1,1,1\], but got \[1,1,1,1,2\]"):
gen_nn_ops.max_pool3d_grad_grad(
orig_in, orig_out, grad, ksize=[1, 1, 1, 1, 1],
strides=[1, 1, 1, 1, 1], padding="VALID")
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/pooling_ops_test.py
+2669
View File
File diff suppressed because it is too large Load Diff
tensorflow/python/kernel_tests/nn_ops/relu_op_test.py
+683
View File
@@ -0,0 +1,683 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for Relu and ReluGrad."""
import numpy as np
from tensorflow.python.eager import backprop
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import gradient_checker_v2
from tensorflow.python.ops import nn_ops
from tensorflow.python.ops import random_ops
from tensorflow.python.ops import variables
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
from tensorflow.python.training import gradient_descent
def _elu_grad_grad(activation):
if activation < 0:
return np.exp(activation)
return 0
class ReluTest(test.TestCase):
def _npRelu(self, np_features):
return np.maximum(np_features, np.zeros(np_features.shape))
def testNpRelu(self):
self.assertAllClose(
np.array([[0.0, 0.7, 0.0, 0.3, 0.0], [0.1, 0.0, 0.5, 0.0, 0.9]]),
self._npRelu(
np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7,
0.9]])))
def _testRelu(self, np_features):
np_relu = self._npRelu(np_features)
tf_relu = nn_ops.relu(np_features)
self.assertAllClose(np_relu, tf_relu)
self.assertShapeEqual(np_relu, tf_relu)
def testNumbersCPU(self):
for t in [
np.int32, np.int64, np.float16, np.float32, np.float64,
dtypes.bfloat16.as_numpy_dtype
]:
# Force execution on CPU even if a GPU kernel is available for the type.
with ops.device("/device:CPU:0"):
self._testRelu(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
def testNumbersGPU(self):
if not test.is_gpu_available():
self.skipTest("No GPU available")
for t in [
np.float16,
np.float32,
np.float64,
dtypes.bfloat16.as_numpy_dtype,
]:
self._testRelu(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
def testReluInt8x4GoodShape(self):
if not test.is_gpu_available(cuda_only=True):
self.skipTest("No GPU available")
inputs = np.array([[-50, 7, 23, 0], [-1, -5, 6, 11]])
np_relu = self._npRelu(inputs)
tf_relu = nn_ops.relu(constant_op.constant(inputs, dtypes.qint8))
self.assertAllClose(np_relu, tf_relu)
self.assertShapeEqual(np_relu, tf_relu)
@test_util.disable_xla("b/123338077") # Passes with XLA
def testReluInt8x4BadShape(self):
if not test.is_gpu_available(cuda_only=True):
self.skipTest("No GPU available")
inputs = constant_op.constant(
np.array([[-50, 7, 23], [0, 1, -5], [6, -2, 11]]), dtypes.qint8)
with self.assertRaisesRegex(
errors.InvalidArgumentError,
"Tensor size must be a multiple of 4 for Relu<qint8>. Got 9"):
self.evaluate(nn_ops.relu(inputs))
inputs = constant_op.constant(
np.array([1, -2, 3, -4, 5, -6, 7, -8, 9, -8, 7, -6, 5, -4, 3, -2, 1]),
dtypes.qint8)
with self.assertRaisesRegex(
errors.InvalidArgumentError,
"Tensor size must be a multiple of 4 for Relu<qint8>. Got 17"):
self.evaluate(nn_ops.relu(inputs))
def testNoElement(self):
self._testRelu(np.array([[], []], dtype=np.float32))
@test_util.disable_xla("b/157978028: Does not yet pass with XLA")
def testNaNPropagation(self):
for t in [np.float16, np.float32, np.float64]:
self._testRelu(np.array([-1, np.nan, 1, np.nan]).astype(t))
# The gradient test for ReLU is a bit tricky as the derivative is not well
# defined at around zero and we want to avoid that in terms of input values.
def testGradientFloat32(self):
with self.cached_session():
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float32,
order="F")
err = gradient_checker_v2.max_error(*gradient_checker_v2.compute_gradient(
nn_ops.relu, [x], delta=1.0 / 1024))
self.assertLess(err, 1e-6)
# The gradient test for ReLU is a bit tricky as the derivative is not well
# defined at around zero and we want to avoid that in terms of input values.
def testGradientFloat16(self):
with self.cached_session():
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float16,
order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(nn_ops.relu, [x]))
self.assertLess(err, 1e-6)
def testGradientFloat64(self):
with self.cached_session():
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float64,
order="F")
err = gradient_checker_v2.max_error(*gradient_checker_v2.compute_gradient(
nn_ops.relu, [x], delta=1.0 / 1024))
self.assertLess(err, 1e-15)
def testGradGradFloat32(self):
with self.cached_session():
def f(x):
assert x.dtype == dtypes.float32
with backprop.GradientTape() as tape:
tape.watch(x)
y = nn_ops.relu(x)
return tape.gradient(y, x)
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float32,
order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(f, [x], delta=1.0 / 1024))
self.assertLess(err, 1e-4)
def testGradGradFloat64(self):
with self.cached_session():
def f(x):
assert x.dtype == dtypes.float64
with backprop.GradientTape() as tape:
tape.watch(x)
y = nn_ops.relu(x)
return tape.gradient(y, x)
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float64,
order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(f, [x], delta=1.0 / 1024))
self.assertLess(err, 1e-10)
def testGradientScalar(self):
x = variables.Variable(100.)
def loss():
return nn_ops.relu(x)**2
optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.25)
self.evaluate(variables.global_variables_initializer())
self.evaluate(optimizer.minimize(loss))
self.assertAllClose(x.read_value(), 50.0)
def testGradientNoElement(self):
with self.cached_session():
def f(x):
with backprop.GradientTape() as tape:
tape.watch(x)
y = nn_ops.relu(x)
return tape.gradient(y, x)
x = np.asarray([[], []], dtype=np.float32)
z = list(gradient_checker_v2.compute_gradient(f, [x]))[0][0]
self.assertAllEqual(z, np.reshape(x, (0, 0)))
class Relu6Test(test.TestCase):
def _npRelu6(self, np_features):
sixes = np.copy(np_features)
sixes.fill(6.0)
return np.minimum(
np.maximum(np_features, np.zeros(np_features.shape)), sixes)
def testNpRelu6(self):
self.assertAllClose(
np.array([[0.0, 0.7, 0.0, 0.3, 6.0], [0.1, 0.0, 6.0, 0.0, 0.9]]),
self._npRelu6(
np.array([[-0.9, 0.7, -0.5, 0.3, 6.0], [0.1, -0.3, 6.5, -0.7,
0.9]])))
def _testRelu6(self, np_features):
np_relu6 = self._npRelu6(np_features)
tf_relu6 = nn_ops.relu6(np_features)
self.assertAllClose(np_relu6, tf_relu6)
self.assertShapeEqual(np_relu6, tf_relu6)
def testNumbersCPU(self):
for t in [
np.int32, np.int64, np.float16, np.float32, np.float64,
dtypes.bfloat16.as_numpy_dtype
]:
# Force execution on CPU even if a GPU kernel is available for the type.
with ops.device("/device:CPU:0"):
self._testRelu6(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
def testNumbersGPU(self):
if not test.is_gpu_available():
self.skipTest("No GPU available")
for t in [
np.float16,
np.float32,
np.float64,
dtypes.bfloat16.as_numpy_dtype,
]:
self._testRelu6(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
@test_util.disable_xla("b/157978028: Does not yet pass with XLA")
def testNaNPropagation(self):
for t in [np.float16, np.float32, np.float64]:
self._testRelu6(np.array([-1, np.nan, 1, 7, np.nan]).astype(t))
# The gradient test for ReLU6 is a bit tricky as the derivative is
# not well defined at around zero and six and we want to avoid that
# in terms of input values.
def testGradientFloat32(self):
with self.cached_session():
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [6.1, 6.3, 6.5, 6.7, 6.9]],
dtype=np.float32,
order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(nn_ops.relu6, [x]))
self.assertLess(err, 1e-4)
def testGradientFloat64(self):
with self.cached_session():
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [6.1, 6.3, 6.5, 6.7, 6.9]],
dtype=np.float64,
order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(nn_ops.relu6, [x]))
self.assertLess(err, 1e-10)
class LeakyReluTest(test.TestCase):
def _npLeakyRelu(self, np_features, alpha=0.1):
return np.maximum(np_features, alpha * np_features)
def testNpLeakyRelu(self):
self.assertAllClose(
np.array([[-0.09, 0.7, -0.05, 0.3, -0.01],
[0.1, -0.03, 0.5, -0.07, 0.9]]),
self._npLeakyRelu(
np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7,
0.9]]),
alpha=0.1))
def _testLeakyRelu(self, np_features, alpha):
np_leaky_relu = self._npLeakyRelu(np_features, alpha)
tf_leaky_relu = nn_ops.leaky_relu(np_features, alpha)
self.assertAllCloseAccordingToType(np_leaky_relu, tf_leaky_relu)
self.assertShapeEqual(np_leaky_relu, tf_leaky_relu)
def testNumbersCPU(self):
for t in [
np.int32, np.int64, np.float16, np.float32, np.float64,
dtypes.bfloat16.as_numpy_dtype
]:
# Force execution on CPU even if a GPU kernel is available for the type.
with ops.device("/device:CPU:0"):
self._testLeakyRelu(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
alpha=0.2)
def testNumbersGPU(self):
if not test.is_gpu_available():
self.skipTest("No GPU available")
for t in [
np.float16,
np.float32,
np.float64,
dtypes.bfloat16.as_numpy_dtype,
]:
self._testLeakyRelu(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
alpha=0.1)
def testNaNPropagation(self):
for t in [np.float16, np.float32, np.float64]:
self._testLeakyRelu(np.array([-1, np.nan, 1, np.nan]).astype(t),
alpha=0.2)
# The gradient test for Leaky ReLU is a bit tricky as the derivative is not
# well defined at around zero and we want to avoid that in terms of input
# values.
def testGradientFloat32(self):
with self.cached_session():
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float32,
order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(nn_ops.leaky_relu, [x]))
self.assertLess(err, 1e-4)
def testGradientFloat64(self):
with self.cached_session():
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float64,
order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(nn_ops.leaky_relu, [x]))
self.assertLess(err, 1e-10)
def testGradGradFloat32(self):
with self.cached_session():
def f(x):
assert x.dtype == dtypes.float32
with backprop.GradientTape() as tape:
tape.watch(x)
y = nn_ops.leaky_relu(x)
return tape.gradient(y, x)
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float32,
order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(f, [x]))
self.assertLess(err, 1e-4)
def testGradGradFloat64(self):
with self.cached_session():
def f(x):
assert x.dtype == dtypes.float64
with backprop.GradientTape() as tape:
tape.watch(x)
y = nn_ops.leaky_relu(x)
return tape.gradient(y, x)
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float64,
order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(f, [x]))
self.assertLess(err, 1e-10)
def testGradientScalar(self):
x = variables.Variable(-100.)
def loss():
return nn_ops.leaky_relu(x, 0.05)**2
optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.2)
self.evaluate(variables.global_variables_initializer())
self.evaluate(optimizer.minimize(loss))
self.assertAllClose(x.read_value(), -99.9)
def testUnexpectedAlphaValue(self):
self.assertAllClose(
np.array([[-9.0, 0.7, -5.0, 0.3, -0.1], [0.1, -3.0, 0.5, -27.0, 0.9]]),
nn_ops.leaky_relu(
np.array([[-0.9, 0.7, -0.5, 0.3, -0.01],
[0.1, -0.3, 0.5, -2.7, 0.9]]),
alpha=10))
self.assertAllClose(
np.array([[9.0, 0.7, 5.0, 0.3, 0.1], [0.1, 3.0, 0.5, 27.0, 0.9]]),
nn_ops.leaky_relu(
np.array([[-0.9, 0.7, -0.5, 0.3, -0.01],
[0.1, -0.3, 0.5, -2.7, 0.9]]),
alpha=-10))
class EluTest(test.TestCase):
def _npElu(self, np_features):
return np.where(np_features < 0, np.exp(np_features) - 1, np_features)
def testNpElu(self):
self.assertAllClose(
np.array([[-0.59343034025, 0.7, -0.39346934028, 0.3, -0.09516258196],
[0.1, -0.25918177931, 0.5, -0.5034146962, 0.9]]),
self._npElu(
np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7,
0.9]])))
def _testElu(self, np_features):
np_elu = self._npElu(np_features)
tf_elu = nn_ops.elu(np_features)
self.assertAllCloseAccordingToType(np_elu, tf_elu)
self.assertShapeEqual(np_elu, tf_elu)
def testNumbersCPU(self):
for t in [
np.float16, np.float32, np.float64, dtypes.bfloat16.as_numpy_dtype
]:
# Force execution on CPU even if a GPU kernel is available for the type.
with ops.device("/device:CPU:0"):
self._testElu(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
def testNumbersGPU(self):
if not test.is_gpu_available():
self.skipTest("No GPU available")
for t in [
np.float16,
np.float32,
np.float64,
dtypes.bfloat16.as_numpy_dtype,
]:
self._testElu(np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
def testNaNPropagation(self):
for t in [np.float16, np.float32, np.float64]:
self._testElu(np.array([-1, np.nan, 1, np.nan]).astype(t))
def testGradientFloat32(self):
with self.cached_session():
x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]]
x = np.asarray(x_val, dtype=np.float32, order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(nn_ops.elu, [x]))
self.assertLess(err, 1e-4)
def testGradientFloat64(self):
with self.cached_session():
x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]]
x = np.asarray(x_val, dtype=np.float64, order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(nn_ops.elu, [x]))
self.assertLess(err, 1e-6)
def testGradGrad(self):
with self.cached_session():
def f(x):
with backprop.GradientTape(persistent=True) as tape:
tape.watch(x)
y = nn_ops.elu(x)
dy = tape.gradient(y, x)
return tape.gradient(dy, x)
for x in [-1., -0.5, 0.5, 1.]:
got = self.evaluate(f(constant_op.constant(x)))
want = _elu_grad_grad(x)
err = np.abs(got - want)
self.assertLess(err, 1e-4)
def testGradGradFloat32(self):
with self.cached_session():
def f(x):
assert x.dtype == dtypes.float32
with backprop.GradientTape() as tape:
tape.watch(x)
y = nn_ops.elu(x)
return tape.gradient(y, x)
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float32,
order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(f, [x]))
self.assertLess(err, 1e-4)
def testGradGradFloat64(self):
with self.cached_session():
def f(x):
assert x.dtype == dtypes.float64
with backprop.GradientTape() as tape:
tape.watch(x)
y = nn_ops.elu(x)
return tape.gradient(y, x)
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float64,
order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(f, [x]))
self.assertLess(err, 1e-6)
class SeluTest(test.TestCase):
def _npSelu(self, np_features):
scale = 1.0507009873554804934193349852946
scale_alpha = 1.7580993408473768599402175208123
return np.where(np_features < 0, scale_alpha * (np.exp(np_features) - 1),
scale * np_features)
def testNpSelu(self):
self.assertAllClose(
np.array([[-1.0433095, 0.73549069, -0.6917582, 0.3152103, -0.16730527],
[0.1050701, -0.45566732, 0.5253505, -0.88505305, 0.9456309]]),
self._npSelu(
np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7,
0.9]])))
def _testSelu(self, np_features):
np_selu = self._npSelu(np_features)
tf_selu = nn_ops.selu(np_features)
self.assertAllCloseAccordingToType(np_selu, tf_selu)
self.assertShapeEqual(np_selu, tf_selu)
def testNumbers(self):
for t in [
np.float16, np.float32, np.float64, dtypes.bfloat16.as_numpy_dtype
]:
self._testSelu(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
# Force executed on CPU in case GPU kernels are available.
with ops.device("/device:CPU:0"):
self._testSelu(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
def testNaNPropagation(self):
for t in [
np.float16,
np.float32,
np.float64,
dtypes.bfloat16.as_numpy_dtype,
]:
self._testSelu(np.array([-1, np.nan, 1, np.nan]).astype(t))
# Force executed on CPU in case GPU kernels are available.
with ops.device("/device:CPU:0"):
self._testSelu(np.array([-1, np.nan, 1, np.nan]).astype(t))
def testGradientFloat32(self):
with self.cached_session():
x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]]
x = np.asarray(x_val, dtype=np.float32, order="F")
err = gradient_checker_v2.max_error(*gradient_checker_v2.compute_gradient(
nn_ops.selu, [x], delta=1.0 / 1024))
self.assertLess(err, 1e-4)
def testGradientFloat64(self):
with self.cached_session():
x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]]
x = np.asarray(x_val, dtype=np.float64, order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(nn_ops.selu, [x]))
self.assertLess(err, 1e-6)
def testGradGradFloat32(self):
with self.cached_session():
def f(x):
assert x.dtype == dtypes.float32
with backprop.GradientTape() as tape:
tape.watch(x)
y = nn_ops.selu(x)
return tape.gradient(y, x)
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float32,
order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(f, [x], delta=1.0 / 1024))
self.assertLess(err, 1e-4)
def testGradGradFloat64(self):
with self.cached_session():
def f(x):
assert x.dtype == dtypes.float64
with backprop.GradientTape() as tape:
tape.watch(x)
y = nn_ops.selu(x)
return tape.gradient(y, x)
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float64,
order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(f, [x]))
self.assertLess(err, 1e-6)
class CreluTest(test.TestCase):
def testCreluShape(self):
f = random_ops.random_normal([50, 5, 7, 10])
t = nn_ops.crelu(f)
self.assertEqual([50, 5, 7, 20], t.get_shape())
def _testCrelu(self, np_features):
np_relu = np.maximum(np_features, np.zeros_like(np_features))
np_neg_relu = np.maximum(-np_features, np.zeros_like(np_features))
np_crelu = np.concatenate((np_relu, np_neg_relu),
len(np_features.shape) - 1)
tf_crelu = nn_ops.crelu(np_features)
self.assertAllClose(np_crelu, tf_crelu)
self.assertShapeEqual(np_crelu, tf_crelu)
def testNumbersCPU(self):
for t in [
np.int32, np.int64, np.float16, np.float32, np.float64,
dtypes.bfloat16.as_numpy_dtype
]:
# Force execution on CPU even if a GPU kernel is available for the type.
with ops.device("/device:CPU:0"):
self._testCrelu(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
def testNumbersGPU(self):
if not test.is_gpu_available():
self.skipTest("No GPU available")
for t in [
np.float16,
np.float32,
np.float64,
dtypes.bfloat16.as_numpy_dtype,
]:
self._testCrelu(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
def testNumbersWithAxis0(self):
tf_crelu = nn_ops.crelu(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]), axis=0)
np_crelu = np.array([[0, 7, 0, 3, 0], [1, 0, 5, 0, 9], [9, 0, 5, 0, 1],
[0, 3, 0, 7, 0]])
self.assertAllEqual(np_crelu, tf_crelu)
def testNumbersWithAxis1(self):
tf_crelu = nn_ops.crelu(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]), axis=1)
np_crelu = np.array([[0, 7, 0, 3, 0, 9, 0, 5, 0, 1],
[1, 0, 5, 0, 9, 0, 3, 0, 7, 0]])
self.assertAllEqual(np_crelu, tf_crelu)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/rnn_cell_test.py
+3514
View File
File diff suppressed because it is too large Load Diff
tensorflow/python/kernel_tests/nn_ops/rnn_test.py
+847
View File
@@ -0,0 +1,847 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for rnn module."""
import os
import time
import timeit
import numpy as np
from tensorflow.core.protobuf import config_pb2
from tensorflow.python.client import session
from tensorflow.python.eager import context
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops as ops_lib
from tensorflow.python.framework import tensor_shape
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import array_ops_stack
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import gradients_impl
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import rnn
from tensorflow.python.ops import rnn_cell_impl
from tensorflow.python.ops import tensor_array_ops
from tensorflow.python.ops import variables as variables_lib
import tensorflow.python.ops.data_flow_grad # pylint: disable=unused-import
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
import tensorflow.python.ops.sparse_grad # pylint: disable=unused-import
import tensorflow.python.ops.tensor_array_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
from tensorflow.python.training import saver
class Plus1RNNCell(rnn_cell_impl.RNNCell):
"""RNN Cell generating (output, new_state) = (input + 1, state + 1)."""
@property
def output_size(self):
return 5
@property
def state_size(self):
return 5
def call(self, input_, state, scope=None):
return (input_ + 1, state + 1)
class ScalarStateRNNCell(rnn_cell_impl.RNNCell):
"""RNN Cell generating (output, new_state) = (input + 1, state + 1)."""
@property
def output_size(self):
return 1
@property
def state_size(self):
return tensor_shape.TensorShape([])
def zero_state(self, batch_size, dtype):
return array_ops.zeros([], dtype=dtypes.int32)
def call(self, input_, state, scope=None):
return (input_, state + 1)
class UnbalancedOutputRNNCell(rnn_cell_impl.RNNCell):
"""RNN Cell generating (output, new_state) = (input + 1, state + 1)."""
@property
def output_size(self):
return tensor_shape.TensorShape(1), tensor_shape.TensorShape((2))
@property
def state_size(self):
return tensor_shape.TensorShape([])
def zero_state(self, batch_size, dtype):
return array_ops.zeros([], dtype=dtypes.int32)
def call(self, input_, state, scope=None):
concatenated = array_ops.concat((input_, input_), axis=-1)
return (input_, concatenated), state + 1
class TensorArrayStateRNNCell(rnn_cell_impl.RNNCell):
"""RNN Cell its state as a TensorArray."""
@property
def output_size(self):
return 1
@property
def state_size(self):
return (tensor_shape.TensorShape([]), ())
def zero_state(self, batch_size, dtype):
return (array_ops.zeros([], dtype=dtypes.int32),
tensor_array_ops.TensorArray(
dtype=dtype, size=0, dynamic_size=True))
def call(self, input_, state, scope=None):
new_array = state[1].write(state[0], input_)
return (input_, (state[0] + 1, new_array))
class RNNTest(test.TestCase):
def setUp(self):
self._seed = 23489
np.random.seed(self._seed)
@test_util.run_in_graph_and_eager_modes
def testInvalidSequenceLengthShape(self):
cell = Plus1RNNCell()
if context.executing_eagerly():
inputs = [constant_op.constant(np.ones((3, 4)))]
else:
inputs = [array_ops.placeholder(dtypes.float32, shape=(3, 4))]
with self.assertRaisesRegex(ValueError, "must be a vector"):
rnn.dynamic_rnn(
cell,
array_ops_stack.stack(inputs),
dtype=dtypes.float32,
sequence_length=[[4]])
@test_util.run_in_graph_and_eager_modes
def testInvalidDtype(self):
if context.executing_eagerly():
inputs = np.zeros((3, 4, 5), dtype=np.int32)
else:
inputs = array_ops.placeholder(dtypes.int32, shape=(3, 4, 5))
cells = [
rnn_cell_impl.BasicRNNCell,
rnn_cell_impl.GRUCell,
rnn_cell_impl.BasicLSTMCell,
rnn_cell_impl.LSTMCell,
]
for cell_cls in cells:
with self.cached_session():
with self.assertRaisesRegex(ValueError,
"RNN cell only supports floating"):
cell = cell_cls(2, dtype=dtypes.int32)
rnn.dynamic_rnn(cell, inputs, dtype=dtypes.int32)
@test_util.run_in_graph_and_eager_modes
def testBatchSizeFromInput(self):
cell = Plus1RNNCell()
in_eager_mode = context.executing_eagerly()
# With static batch size
if in_eager_mode:
inputs = np.zeros((3, 4, 5), dtype=np.float32)
initial_state = np.zeros((3, 5), dtype=np.float32)
else:
inputs = array_ops.placeholder(dtypes.float32, shape=(3, 4, 5))
initial_state = array_ops.placeholder(dtypes.float32, shape=(3, 5))
# - Without initial_state
outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32)
self.assertEqual(3, outputs.shape[0])
self.assertEqual(3, state.shape[0])
# - With initial_state
outputs, state = rnn.dynamic_rnn(
cell, inputs, initial_state=initial_state)
self.assertEqual(3, outputs.shape[0])
self.assertEqual(3, state.shape[0])
# Without static batch size
# Tensor shapes are fully determined with eager execution enabled,
# so only run this test for graph construction.
if not in_eager_mode:
inputs = array_ops.placeholder(dtypes.float32, shape=(None, 4, 5))
# - Without initial_state
outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32)
self.assertEqual(None, outputs.shape.dims[0].value)
self.assertEqual(None, state.shape.dims[0].value)
# - With initial_state
outputs, state = rnn.dynamic_rnn(
cell,
inputs,
initial_state=array_ops.placeholder(dtypes.float32, shape=(None, 5)))
self.assertEqual(None, outputs.shape.dims[0].value)
self.assertEqual(None, state.shape.dims[0].value)
@test_util.run_in_graph_and_eager_modes
def testScalarStateIsAccepted(self):
cell = ScalarStateRNNCell()
in_eager_mode = context.executing_eagerly()
if in_eager_mode:
inputs = np.array([[[1], [2], [3], [4]]], dtype=np.float32)
else:
inputs = array_ops.placeholder(dtypes.float32, shape=(1, 4, 1))
with self.cached_session() as sess:
outputs, state = rnn.dynamic_rnn(
cell, inputs, dtype=dtypes.float32, sequence_length=[4])
if not in_eager_mode:
outputs, state = sess.run(
[outputs, state], feed_dict={inputs: [[[1], [2], [3], [4]]]})
self.assertAllEqual([[[1], [2], [3], [4]]], outputs)
self.assertAllEqual(4, state)
@test_util.run_in_graph_and_eager_modes
def testUnbalancedOutputIsAccepted(self):
cell = UnbalancedOutputRNNCell()
in_eager_mode = context.executing_eagerly()
if in_eager_mode:
inputs = np.array([[[1], [2], [3], [4]]], dtype=np.float32)
else:
inputs = array_ops.placeholder(dtypes.float32, shape=(1, 4, 1))
with self.cached_session() as sess:
outputs, state = rnn.dynamic_rnn(
cell, inputs, dtype=dtypes.float32, sequence_length=[4])
if not in_eager_mode:
outputs, state = sess.run(
[outputs, state], feed_dict={inputs: [[[1], [2], [3], [4]]]})
self.assertIsInstance(outputs, tuple)
self.assertAllEqual([[[1], [2], [3], [4]]], outputs[0])
self.assertAllEqual([[[1, 1], [2, 2], [3, 3], [4, 4]]], outputs[1])
self.assertAllEqual(4, state)
@test_util.assert_no_new_pyobjects_executing_eagerly()
def testEagerMemory(self):
with context.eager_mode():
cell = TensorArrayStateRNNCell()
inputs = np.array([[[1], [2], [3], [4]]], dtype=np.float32)
rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32, sequence_length=[4])
@test_util.run_in_graph_and_eager_modes
@test_util.run_v1_only("b/120545219")
def testTensorArrayStateIsAccepted(self):
cell = TensorArrayStateRNNCell()
in_eager_mode = context.executing_eagerly()
if in_eager_mode:
inputs = np.array([[[1], [2], [3], [4]]], dtype=np.float32)
else:
inputs = array_ops.placeholder(dtypes.float32, shape=(1, 4, 1))
with self.cached_session() as sess:
outputs, state = rnn.dynamic_rnn(
cell, inputs, dtype=dtypes.float32, sequence_length=[4])
state = (state[0], state[1].stack())
if not in_eager_mode:
outputs, state = sess.run(
[outputs, state], feed_dict={
inputs: [[[1], [2], [3], [4]]]
})
self.assertAllEqual([[[1], [2], [3], [4]]], outputs)
self.assertAllEqual(4, state[0])
self.assertAllEqual([[[1]], [[2]], [[3]], [[4]]], state[1])
@test_util.run_deprecated_v1
def testCellGetInitialState(self):
cell = rnn_cell_impl.BasicRNNCell(5)
with self.assertRaisesRegex(ValueError,
"batch_size and dtype cannot be None"):
cell.get_initial_state(None, None, None)
inputs = array_ops.placeholder(dtypes.float32, shape=(None, 4, 1))
with self.assertRaisesRegex(
ValueError, "batch size from input tensor is different from"):
cell.get_initial_state(inputs=inputs, batch_size=50, dtype=None)
with self.assertRaisesRegex(
ValueError, "batch size from input tensor is different from"):
cell.get_initial_state(
inputs=inputs, batch_size=constant_op.constant(50), dtype=None)
with self.assertRaisesRegex(ValueError,
"dtype from input tensor is different from"):
cell.get_initial_state(inputs=inputs, batch_size=None, dtype=dtypes.int16)
initial_state = cell.get_initial_state(
inputs=inputs, batch_size=None, dtype=None)
self.assertEqual(initial_state.shape.as_list(), [None, 5])
self.assertEqual(initial_state.dtype, inputs.dtype)
batch = array_ops.shape(inputs)[0]
dtype = inputs.dtype
initial_state = cell.get_initial_state(None, batch, dtype)
self.assertEqual(initial_state.shape.as_list(), [None, 5])
self.assertEqual(initial_state.dtype, inputs.dtype)
def _assert_cell_builds(self, cell_class, dtype, batch_size, in_size,
out_size):
cell = cell_class(out_size, dtype=dtype)
in_shape = tensor_shape.TensorShape((batch_size, in_size))
cell.build(in_shape)
state_output = cell.get_initial_state(
inputs=None, batch_size=batch_size, dtype=dtype)
cell_output, _ = cell(array_ops.zeros(in_shape, dtype), state_output)
self.assertAllEqual([batch_size, out_size], cell_output.shape.as_list())
@test_util.run_in_graph_and_eager_modes
def testCellsBuild(self):
f32 = dtypes.float32
f64 = dtypes.float64
self._assert_cell_builds(rnn_cell_impl.BasicRNNCell, f32, 5, 7, 3)
self._assert_cell_builds(rnn_cell_impl.BasicRNNCell, f64, 5, 7, 3)
self._assert_cell_builds(rnn_cell_impl.BasicLSTMCell, f32, 5, 7, 3)
self._assert_cell_builds(rnn_cell_impl.BasicLSTMCell, f64, 5, 7, 3)
self._assert_cell_builds(rnn_cell_impl.GRUCell, f32, 5, 7, 3)
self._assert_cell_builds(rnn_cell_impl.GRUCell, f64, 5, 7, 3)
self._assert_cell_builds(rnn_cell_impl.LSTMCell, f32, 5, 7, 3)
self._assert_cell_builds(rnn_cell_impl.LSTMCell, f64, 5, 7, 3)
@test_util.run_deprecated_v1
def testBasicLSTMCellInterchangeWithLSTMCell(self):
with self.session(graph=ops_lib.Graph()) as sess:
basic_cell = rnn_cell_impl.BasicLSTMCell(1)
basic_cell(array_ops.ones([1, 1]),
state=basic_cell.get_initial_state(inputs=None,
batch_size=1,
dtype=dtypes.float32))
self.evaluate([v.initializer for v in basic_cell.variables])
self.evaluate(basic_cell._bias.assign([10.] * 4))
save = saver.Saver()
prefix = os.path.join(self.get_temp_dir(), "ckpt")
save_path = save.save(sess, prefix)
with self.session(graph=ops_lib.Graph()) as sess:
lstm_cell = rnn_cell_impl.LSTMCell(1, name="basic_lstm_cell")
lstm_cell(array_ops.ones([1, 1]),
state=lstm_cell.get_initial_state(inputs=None,
batch_size=1,
dtype=dtypes.float32))
self.evaluate([v.initializer for v in lstm_cell.variables])
save = saver.Saver()
save.restore(sess, save_path)
self.assertAllEqual([10.] * 4, self.evaluate(lstm_cell._bias))
######### Benchmarking RNN code
def _static_vs_dynamic_rnn_benchmark_static(inputs_list_t, sequence_length):
(_, input_size) = inputs_list_t[0].get_shape().as_list()
initializer = init_ops.random_uniform_initializer(-0.01, 0.01, seed=127)
cell = rnn_cell_impl.LSTMCell(
num_units=input_size,
use_peepholes=True,
initializer=initializer,
state_is_tuple=False)
outputs, final_state = rnn.static_rnn(
cell,
inputs_list_t,
sequence_length=sequence_length,
dtype=dtypes.float32)
trainable_variables = ops_lib.get_collection(
ops_lib.GraphKeys.TRAINABLE_VARIABLES)
gradients = gradients_impl.gradients(outputs + [final_state],
trainable_variables)
return control_flow_ops.group(final_state, *(gradients + outputs))
def _static_vs_dynamic_rnn_benchmark_dynamic(inputs_t, sequence_length):
(unused_0, unused_1, input_size) = inputs_t.get_shape().as_list()
initializer = init_ops.random_uniform_initializer(-0.01, 0.01, seed=127)
cell = rnn_cell_impl.LSTMCell(
num_units=input_size,
use_peepholes=True,
initializer=initializer,
state_is_tuple=False)
outputs, final_state = rnn.dynamic_rnn(
cell, inputs_t, sequence_length=sequence_length, dtype=dtypes.float32)
trainable_variables = ops_lib.get_collection(
ops_lib.GraphKeys.TRAINABLE_VARIABLES)
gradients = gradients_impl.gradients([outputs, final_state],
trainable_variables)
return control_flow_ops.group(final_state, outputs, *gradients)
def graph_creation_static_vs_dynamic_rnn_benchmark(max_time):
config = config_pb2.ConfigProto()
config.allow_soft_placement = True
# These parameters don't matter
batch_size = 512
num_units = 512
# Set up sequence lengths
np.random.seed([127])
sequence_length = np.random.randint(0, max_time, size=batch_size)
inputs_list = [
np.random.randn(batch_size, num_units).astype(np.float32)
for _ in range(max_time)
]
inputs = np.dstack(inputs_list).transpose([0, 2, 1]) # batch x time x depth
def _create_static_rnn():
with session.Session(config=config, graph=ops_lib.Graph()):
inputs_list_t = [
variables_lib.Variable(
x, trainable=False).value() for x in inputs_list
]
_static_vs_dynamic_rnn_benchmark_static(inputs_list_t, sequence_length)
def _create_dynamic_rnn():
with session.Session(config=config, graph=ops_lib.Graph()):
inputs_t = variables_lib.Variable(inputs, trainable=False).value()
_static_vs_dynamic_rnn_benchmark_dynamic(inputs_t, sequence_length)
delta_static = timeit.timeit(_create_static_rnn, number=5)
delta_dynamic = timeit.timeit(_create_dynamic_rnn, number=5)
print("%d \t %f \t %f \t %f" %
(max_time, delta_static, delta_dynamic, delta_dynamic / delta_static))
return delta_static, delta_dynamic
def _timer(sess, ops):
# Warm in
for _ in range(2):
sess.run(ops)
# Timing run
runs = 20
start = time.time()
for _ in range(runs):
sess.run(ops)
end = time.time()
return (end - start) / float(runs)
def static_vs_dynamic_rnn_benchmark(batch_size, max_time, num_units, use_gpu):
config = config_pb2.ConfigProto()
config.allow_soft_placement = True
# Set up sequence lengths
np.random.seed([127])
sequence_length = np.random.randint(0, max_time, size=batch_size)
inputs_list = [
np.random.randn(batch_size, num_units).astype(np.float32)
for _ in range(max_time)
]
inputs = np.dstack(inputs_list).transpose([0, 2, 1]) # batch x time x depth
# Using rnn()
with session.Session(config=config, graph=ops_lib.Graph()) as sess:
with ops_lib.device("/cpu:0" if not use_gpu else None):
inputs_list_t = [
variables_lib.Variable(
x, trainable=False).value() for x in inputs_list
]
ops = _static_vs_dynamic_rnn_benchmark_static(inputs_list_t,
sequence_length)
variables_lib.global_variables_initializer().run()
delta_static = _timer(sess, ops)
# Using dynamic_rnn()
with session.Session(config=config, graph=ops_lib.Graph()) as sess:
with ops_lib.device("/cpu:0" if not use_gpu else None):
inputs_t = variables_lib.Variable(inputs, trainable=False).value()
ops = _static_vs_dynamic_rnn_benchmark_dynamic(inputs_t, sequence_length)
variables_lib.global_variables_initializer().run()
delta_dynamic = _timer(sess, ops)
print("%d \t %d \t %d \t %s \t %f \t %f \t %f" %
(batch_size, max_time, num_units, use_gpu, delta_static, delta_dynamic,
delta_dynamic / delta_static))
return delta_static, delta_dynamic
def _half_seq_len_vs_unroll_half_rnn_benchmark(inputs_list_t, sequence_length):
(_, input_size) = inputs_list_t[0].get_shape().as_list()
initializer = init_ops.random_uniform_initializer(-0.01, 0.01, seed=127)
cell = rnn_cell_impl.LSTMCell(
num_units=input_size,
use_peepholes=True,
initializer=initializer,
state_is_tuple=False)
outputs, final_state = rnn.static_rnn(
cell,
inputs_list_t,
sequence_length=sequence_length,
dtype=dtypes.float32)
trainable_variables = ops_lib.get_collection(
ops_lib.GraphKeys.TRAINABLE_VARIABLES)
gradients = gradients_impl.gradients(outputs + [final_state],
trainable_variables)
return control_flow_ops.group(final_state, *(gradients + outputs))
def half_seq_len_vs_unroll_half_rnn_benchmark(batch_size, max_time, num_units,
use_gpu):
config = config_pb2.ConfigProto()
config.allow_soft_placement = True
# Set up sequence lengths
np.random.seed([127])
sequence_length = max_time * np.ones((batch_size,))
inputs_list = [
np.random.randn(batch_size, num_units).astype(np.float32)
for _ in range(max_time)
]
# Halve the sequence length, full static unroll
with session.Session(config=config, graph=ops_lib.Graph()) as sess:
with ops_lib.device("/cpu:0" if not use_gpu else None):
inputs_list_t = [
variables_lib.Variable(
x, trainable=False).value() for x in inputs_list
]
ops = _half_seq_len_vs_unroll_half_rnn_benchmark(inputs_list_t,
sequence_length / 2)
variables_lib.global_variables_initializer().run()
delta_half_seq_len = _timer(sess, ops)
# Halve the unroll size, don't use sequence length
with session.Session(config=config, graph=ops_lib.Graph()) as sess:
with ops_lib.device("/cpu:0" if not use_gpu else None):
inputs_list_t = [
variables_lib.Variable(
x, trainable=False).value() for x in inputs_list
]
ops = _half_seq_len_vs_unroll_half_rnn_benchmark(
inputs_list_t[:(max_time // 2)], sequence_length / 2)
variables_lib.global_variables_initializer().run()
delta_unroll_half = _timer(sess, ops)
print("%d \t %d \t\t %d \t %s \t %f \t\t %f \t\t %f" %
(batch_size, max_time, num_units, use_gpu, delta_half_seq_len,
delta_unroll_half, delta_half_seq_len / delta_unroll_half))
return delta_half_seq_len, delta_unroll_half
def _concat_state_vs_tuple_state_rnn_benchmark(inputs_list_t, sequence_length,
state_is_tuple):
(_, input_size) = inputs_list_t[0].get_shape().as_list()
initializer = init_ops.random_uniform_initializer(-0.01, 0.01, seed=127)
cell = rnn_cell_impl.LSTMCell(
num_units=input_size,
use_peepholes=True,
initializer=initializer,
state_is_tuple=state_is_tuple)
outputs, final_state = rnn.static_rnn(
cell,
inputs_list_t,
sequence_length=sequence_length,
dtype=dtypes.float32)
final_state = list(final_state) if state_is_tuple else [final_state]
trainable_variables = ops_lib.get_collection(
ops_lib.GraphKeys.TRAINABLE_VARIABLES)
gradients = gradients_impl.gradients(outputs + final_state,
trainable_variables)
return control_flow_ops.group(*(final_state + gradients + outputs))
def concat_state_vs_tuple_state_rnn_benchmark(batch_size, max_time, num_units,
use_gpu):
config = config_pb2.ConfigProto()
config.allow_soft_placement = True
# Set up sequence lengths
np.random.seed([127])
sequence_length = max_time * np.ones((batch_size,))
inputs_list = [
np.random.randn(batch_size, num_units).astype(np.float32)
for _ in range(max_time)
]
# Run with concatenated states (default)
with session.Session(config=config, graph=ops_lib.Graph()) as sess:
with ops_lib.device("/cpu:0" if not use_gpu else None):
inputs_list_t = [
variables_lib.Variable(
x, trainable=False).value() for x in inputs_list
]
ops = _concat_state_vs_tuple_state_rnn_benchmark(
inputs_list_t, sequence_length, state_is_tuple=False)
variables_lib.global_variables_initializer().run()
delta_concat_state = _timer(sess, ops)
# Run with tuple states (new)
with session.Session(config=config, graph=ops_lib.Graph()) as sess:
with ops_lib.device("/cpu:0" if not use_gpu else None):
inputs_list_t = [
variables_lib.Variable(
x, trainable=False).value() for x in inputs_list
]
ops = _concat_state_vs_tuple_state_rnn_benchmark(
inputs_list_t, sequence_length, state_is_tuple=True)
variables_lib.global_variables_initializer().run()
delta_tuple_state = _timer(sess, ops)
print("%d \t %d \t %d \t %s \t %f \t\t %f \t\t %f" %
(batch_size, max_time, num_units, use_gpu, delta_concat_state,
delta_tuple_state, delta_concat_state / delta_tuple_state))
return delta_concat_state, delta_tuple_state
def _dynamic_rnn_swap_memory_benchmark(inputs_t, sequence_length, swap_memory):
(unused_0, unused_1, input_size) = inputs_t.get_shape().as_list()
initializer = init_ops.random_uniform_initializer(-0.01, 0.01, seed=127)
cell = rnn_cell_impl.LSTMCell(
num_units=input_size,
use_peepholes=True,
initializer=initializer,
state_is_tuple=False)
outputs, final_state = rnn.dynamic_rnn(
cell,
inputs_t,
sequence_length=sequence_length,
swap_memory=swap_memory,
dtype=dtypes.float32)
trainable_variables = ops_lib.get_collection(
ops_lib.GraphKeys.TRAINABLE_VARIABLES)
gradients = gradients_impl.gradients([outputs, final_state],
trainable_variables)
return control_flow_ops.group(final_state, outputs, *gradients)
def dynamic_rnn_swap_memory_benchmark(batch_size, max_time, num_units):
config = config_pb2.ConfigProto()
config.allow_soft_placement = True
# Set up sequence lengths
np.random.seed([127])
sequence_length = np.random.randint(0, max_time, size=batch_size)
inputs_list = [
np.random.randn(batch_size, num_units).astype(np.float32)
for _ in range(max_time)
]
inputs = np.dstack(inputs_list).transpose([0, 2, 1]) # batch x time x depth
# No memory swap
with session.Session(config=config, graph=ops_lib.Graph()) as sess:
inputs_t = variables_lib.Variable(inputs, trainable=False).value()
ops = _dynamic_rnn_swap_memory_benchmark(
inputs_t, sequence_length, swap_memory=False)
variables_lib.global_variables_initializer().run()
no_swap = _timer(sess, ops)
# Memory swap
with session.Session(config=config, graph=ops_lib.Graph()) as sess:
inputs_t = variables_lib.Variable(inputs, trainable=False).value()
ops = _dynamic_rnn_swap_memory_benchmark(
inputs_t, sequence_length, swap_memory=True)
variables_lib.global_variables_initializer().run()
swap = _timer(sess, ops)
print("%d \t %d \t %d \t %f \t %f \t %f" %
(batch_size, max_time, num_units, no_swap, swap, swap / no_swap))
return no_swap, swap
def rnn_long_sequence_benchmark(batch_size, seqlen, num_units, dynamic,
swap_memory, nn):
config = config_pb2.ConfigProto()
config.allow_soft_placement = True
# Set up sequence lengths
np.random.seed([127])
sequence_length = [seqlen for _ in range(batch_size)]
inputs_list = [
np.random.randn(batch_size, num_units).astype(np.float32)
for _ in range(seqlen)
]
inputs = np.dstack(inputs_list).transpose([0, 2, 1]) # batch x time x depth
for _ in range(nn):
if dynamic:
with session.Session(config=config, graph=ops_lib.Graph()) as sess:
inputs_t = variables_lib.Variable(inputs, trainable=False).value()
ops = _dynamic_rnn_swap_memory_benchmark(
inputs_t, sequence_length, swap_memory=swap_memory)
variables_lib.global_variables_initializer().run()
elapsed = _timer(sess, ops)
else:
with session.Session(config=config, graph=ops_lib.Graph()) as sess:
inputs_list_t = [
variables_lib.Variable(
x, trainable=False).value() for x in inputs_list
]
ops = _static_vs_dynamic_rnn_benchmark_static(inputs_list_t,
sequence_length)
variables_lib.global_variables_initializer().run()
elapsed = _timer(sess, ops)
print("%d \t %d \t %d \t %s \t %f \t %f" % (batch_size, seqlen, num_units,
dynamic, elapsed,
elapsed / seqlen))
class BenchmarkRNN(test.Benchmark):
def benchmarkGraphCreationStaticVsDynamicLSTM(self):
print("Graph Creation: Static Unroll vs. Dynamic Unroll LSTM")
print("max_t \t dt(static) \t dt(dynamic) \t dt(dynamic)/dt(static)")
for max_time in (1, 25, 50):
s_dt, d_dt = graph_creation_static_vs_dynamic_rnn_benchmark(max_time)
self.report_benchmark(
name="graph_creation_time_static_T%02d" % max_time,
iters=5,
wall_time=s_dt)
self.report_benchmark(
name="graph_creation_time_dynamic_T%02d" % max_time,
iters=5,
wall_time=d_dt)
def benchmarkStaticUnrollVsDynamicFlowLSTM(self):
print("Calculation: Static Unroll with Dynamic Flow LSTM "
"vs. Dynamic Unroll LSTM")
print("batch \t max_t \t units \t gpu \t dt(static) \t dt(dynamic) "
"\t dt(dynamic)/dt(static)")
for batch_size in (256,):
for max_time in (50,):
for num_units in (512, 256, 128):
for use_gpu in (False, True):
s_dt, d_dt = static_vs_dynamic_rnn_benchmark(batch_size, max_time,
num_units, use_gpu)
self.report_benchmark(
name="static_unroll_time_T%02d_B%03d_N%03d_gpu_%s" %
(max_time, batch_size, num_units, use_gpu),
iters=20,
wall_time=s_dt)
self.report_benchmark(
name="dynamic_unroll_time_T%02d_B%03d_N%03d_gpu_%s" %
(max_time, batch_size, num_units, use_gpu),
iters=20,
wall_time=d_dt)
def benchmarkDynamicLSTMNoMemorySwapVsMemorySwap(self):
print("Calculation: Dynamic LSTM No Memory Swap vs. Memory Swap")
print("batch \t max_t \t units \t no_swap \t swap \t swap/no_swap")
for batch_size in (256, 512):
for max_time in (100,):
for num_units in (512, 256, 128):
no_swap, swap = dynamic_rnn_swap_memory_benchmark(batch_size,
max_time, num_units)
self.report_benchmark(
name="dynamic_lstm_no_memory_swap_T%02d_B%03d_N%03d" %
(max_time, batch_size, num_units),
iters=20,
wall_time=no_swap)
self.report_benchmark(
name="dynamic_lstm_with_memory_swap_T%02d_B%03d_N%03d" %
(max_time, batch_size, num_units),
iters=20,
wall_time=swap)
def benchmarkStaticUnrollHalfSequenceLengthVsHalfUnroll(self):
print("Calculation: Static Unroll with Halved Sequence Length "
"vs. Half Static Unroll")
print("batch \t full_t \t units \t gpu \t dt(half_seq_len) "
"\t dt(unroll_half) \t dt(half_seq_len)/dt(unroll_half)")
for batch_size in (128,):
for max_time in (50,):
for num_units in (256,):
for use_gpu in (False, True):
s_dt, d_dt = half_seq_len_vs_unroll_half_rnn_benchmark(batch_size,
max_time,
num_units,
use_gpu)
self.report_benchmark(
name="half_seq_len_time_T%02d_B%03d_N%03d_gpu_%s" %
(max_time, batch_size, num_units, use_gpu),
iters=20,
wall_time=s_dt)
self.report_benchmark(
name="unroll_half_time_T%02d_B%03d_N%03d_gpu_%s" %
(max_time, batch_size, num_units, use_gpu),
iters=20,
wall_time=d_dt)
def benchmarkStaticUnrollStateConcatVsStateTuple(self):
print("Calculation: Static Unroll with Concatenated State "
"vs. Tuple State")
print("batch \t time \t units \t gpu \t dt(concat_state) "
"\t dt(tuple_state) \t dt(concat_state)/dt(tuple_state)")
for batch_size in (
16,
128,):
for max_time in (50,):
for num_units in (
16,
128,):
for use_gpu in (False, True):
c_dt, t_dt = concat_state_vs_tuple_state_rnn_benchmark(batch_size,
max_time,
num_units,
use_gpu)
self.report_benchmark(
name="concat_state_time_T%02d_B%03d_N%03d_gpu_%s" %
(max_time, batch_size, num_units, use_gpu),
iters=20,
wall_time=c_dt)
self.report_benchmark(
name="tuple_state_time_T%02d_B%03d_N%03d_gpu_%s" %
(max_time, batch_size, num_units, use_gpu),
iters=20,
wall_time=t_dt)
def _benchmarkDynamicLSTMMemorySwapLongSeq(self):
"""The memory swapping test for the SOSP submission."""
print("Calculation: Long LSTM Sequence")
print("batch \t len \t units \t dynamic \t elapsed_t \t elapsed_t/len")
batch_size = 512
seqlen = 800
num_units = 512
dynamic = True
swap_memory = True
# Some warming up.
if swap_memory:
rnn_long_sequence_benchmark(batch_size, seqlen, num_units,
dynamic, swap_memory, 2)
# Measure the performance.
for slen in range(100, 1100, 100):
rnn_long_sequence_benchmark(batch_size, slen, num_units, dynamic,
swap_memory, 3)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/softmax_op_test.py
+290
View File
@@ -0,0 +1,290 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for SoftmaxOp and LogSoftmaxOp."""
import unittest
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors_impl
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import nn_ops
from tensorflow.python.platform import test
from tensorflow.python.platform import tf_logging as logging
class SoftmaxTest(test.TestCase):
def _npSoftmax(self, features, dim=-1, log=False):
if dim == -1:
dim = len(features.shape) - 1
one_only_on_dim = list(features.shape)
one_only_on_dim[dim] = 1
is_fp16 = features.dtype == np.float16
if is_fp16:
# Do the compute in fp32 and cast the input back to fp32.
features = features.astype(np.float32)
e = np.exp(features - np.reshape(
np.amax(
features, axis=dim), one_only_on_dim))
softmax = e / np.reshape(np.sum(e, axis=dim), one_only_on_dim)
if log:
res = np.log(softmax)
else:
res = softmax
if is_fp16:
res = res.astype(np.float16)
return res
def _testSoftmax(self,
np_features,
dim=-1,
log=False,
dtype=None,
use_gpu=False):
# A previous version of the code checked the op name rather than the op type
# to distinguish between log and non-log. Use an arbitrary name to catch
# this bug in future.
name = "arbitrary"
np_softmax = self._npSoftmax(np_features, dim=dim, log=log)
with self.cached_session(use_gpu=use_gpu):
if dtype is not None:
np_features = math_ops.cast(np_features, dtype=dtype)
if log:
tf_softmax = nn_ops.log_softmax(np_features, axis=dim, name=name)
else:
tf_softmax = nn_ops.softmax(np_features, axis=dim, name=name)
out = self.evaluate(tf_softmax)
self.assertAllCloseAccordingToType(np_softmax, out)
self.assertShapeEqual(np_softmax, tf_softmax)
if not log and dtype is None:
# Bonus check: the softmaxes should add to one in dimension dim.
sum_along_dim = np.sum(out, axis=dim)
self.assertAllCloseAccordingToType(
np.ones(sum_along_dim.shape), sum_along_dim)
def _testAll(self, features, dtype=None):
self._testSoftmax(features, dtype=dtype, use_gpu=True)
self._testSoftmax(features, dtype=dtype, log=True, use_gpu=True)
self._testOverflow(use_gpu=True)
def testNpSoftmax(self):
features = [[1., 1., 1., 1.], [1., 2., 3., 4.]]
# Batch 0: All exps are 1. The expected result is
# Softmaxes = [0.25, 0.25, 0.25, 0.25]
# LogSoftmaxes = [-1.386294, -1.386294, -1.386294, -1.386294]
#
# Batch 1:
# exps = [1., 2.718, 7.389, 20.085]
# sum = 31.192
# Softmaxes = exps / sum = [0.0320586, 0.08714432, 0.23688282, 0.64391426]
# LogSoftmaxes = [-3.44019 , -2.44019 , -1.44019 , -0.44019]
np_sm = self._npSoftmax(np.array(features))
self.assertAllClose(
np.array([[0.25, 0.25, 0.25, 0.25],
[0.0320586, 0.08714432, 0.23688282, 0.64391426]]),
np_sm,
rtol=1.e-5,
atol=1.e-5)
np_lsm = self._npSoftmax(np.array(features), log=True)
self.assertAllClose(
np.array([[-1.386294, -1.386294, -1.386294, -1.386294],
[-3.4401897, -2.4401897, -1.4401897, -0.4401897]]),
np_lsm,
rtol=1.e-5,
atol=1.e-5)
def _testOverflow(self, use_gpu=False):
if use_gpu:
type = np.float32 # pylint: disable=redefined-builtin
else:
type = np.float64 # pylint: disable=redefined-builtin
max = np.finfo(type).max # pylint: disable=redefined-builtin
features = np.array([[1., 1., 1., 1.], [max, 1., 2., 3.]]).astype(type)
with self.cached_session(use_gpu=use_gpu):
tf_log_softmax = nn_ops.log_softmax(features)
out = self.evaluate(tf_log_softmax)
self.assertAllClose(
np.array([[-1.386294, -1.386294, -1.386294, -1.386294],
[0, -max, -max, -max]]),
out,
rtol=1.e-5,
atol=1.e-5)
def testFloat(self):
self._testAll(
np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float32))
@unittest.skipUnless(test.is_built_with_gpu_support(),
"Test only applicable when running on GPUs")
def testFloatGPU(self):
if test.is_gpu_available(cuda_only=True):
rows = [2**x + np.random.randint(0, 16) for x in range(1, 4)]
cols = [2**x + np.random.randint(0, 16) for x in range(1, 4)]
for row, col in zip(rows, cols):
logging.info("Testing softmax float dtype in shape [%d, %d]", row, col)
data = np.random.rand(row, col)
self._testAll(data.astype(np.float32))
def testHalf(self):
self._testAll(
np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float16))
@unittest.skipUnless(test.is_built_with_gpu_support(),
"Test only applicable when running on GPUs")
def testHalfGPU(self):
if test.is_gpu_available(cuda_only=True):
rows = [2**x + np.random.randint(0, 16) for x in range(1, 4)]
cols = [2**x + np.random.randint(0, 16) for x in range(1, 4)]
for row, col in zip(rows, cols):
logging.info("Testing softmax half dtype in shape [%d, %d]", row, col)
data = np.random.rand(row, col)
self._testAll(data.astype(np.float16))
def testDouble(self):
self._testSoftmax(
np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float64))
self._testOverflow()
@unittest.skipUnless(test.is_built_with_gpu_support(),
"Test only applicable when running on GPUs")
def testDoubleGPU(self):
if test.is_gpu_available(cuda_only=True):
rows = [2**x + np.random.randint(0, 16) for x in range(1, 4)]
cols = [2**x + np.random.randint(0, 16) for x in range(1, 4)]
for row, col in zip(rows, cols):
logging.info("Testing softmax float dtype in shape [%d, %d]", row, col)
data = np.random.rand(row, col)
self._testAll(data.astype(np.float64))
def testBfloat16(self):
self._testAll(
np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float32),
dtype=dtypes.bfloat16)
@unittest.skipUnless(test.is_built_with_gpu_support(),
"Test only applicable when running on GPUs")
def testBfloat16GPU(self):
if test.is_gpu_available(cuda_only=True):
rows = [2**x + np.random.randint(0, 16) for x in range(1, 4)]
cols = [2**x + np.random.randint(0, 16) for x in range(1, 4)]
for row, col in zip(rows, cols):
logging.info("Testing softmax bfloat16 dtype in shape [%d, %d]", row,
col)
data = np.random.rand(row, col)
self._testAll(data.astype(dtypes.bfloat16.as_numpy_dtype))
def test1DTensorAsInput(self):
self._testSoftmax(
np.array([3., 2., 3., 9.]).astype(np.float64), use_gpu=False)
self._testOverflow(use_gpu=False)
def test1DTensorAsInputNoReshape(self):
self._testSoftmax(
np.array([3., 2., 3., 9.]).astype(np.float64), use_gpu=False)
self._testOverflow(use_gpu=False)
def test3DTensorAsInput(self):
self._testSoftmax(
np.array([[[1., 1., 1., 1.], [1., 2., 3., 4.]],
[[2., 3., 4., 5.], [6., 7., 8., 9.]],
[[5., 4., 3., 2.], [1., 2., 3., 4.]]]).astype(np.float32),
use_gpu=False)
self._testOverflow(use_gpu=False)
def test3DTensorAsInputNoReshape(self):
self._testSoftmax(
np.array([[[1., 1., 1., 1.], [1., 2., 3., 4.]],
[[2., 3., 4., 5.], [6., 7., 8., 9.]],
[[5., 4., 3., 2.], [1., 2., 3., 4.]]]).astype(np.float32),
use_gpu=False)
self._testOverflow(use_gpu=False)
def testAlongFirstDimension(self):
self._testSoftmax(
np.array([[[1., 1., 1., 1.], [1., 2., 3., 4.]],
[[2., 3., 4., 5.], [6., 7., 8., 9.]],
[[5., 4., 3., 2.], [1., 2., 3., 4.]]]).astype(np.float32),
dim=0,
use_gpu=False)
self._testOverflow(use_gpu=False)
def testAlongSecondDimension(self):
self._testSoftmax(
np.array([[[1., 1., 1., 1.], [1., 2., 3., 4.]],
[[2., 3., 4., 5.], [6., 7., 8., 9.]],
[[5., 4., 3., 2.], [1., 2., 3., 4.]]]).astype(np.float32),
dim=1,
use_gpu=False)
self._testOverflow(use_gpu=False)
def testAlongNegativeDimension(self):
self._testSoftmax(
np.array([[[1., 1., 1., 1.], [1., 2., 3., 4.]],
[[2., 3., 4., 5.], [6., 7., 8., 9.]],
[[5., 4., 3., 2.], [1., 2., 3., 4.]]]).astype(np.float32),
dim=-2,
use_gpu=False)
self._testOverflow(use_gpu=False)
def testShapeInference(self):
op = nn_ops.softmax([[[1., 1., 1., 1.], [1., 2., 3., 4.]],
[[2., 3., 4., 5.], [6., 7., 8., 9.]],
[[5., 4., 3., 2.], [1., 2., 3., 4.]]])
self.assertEqual([3, 2, 4], op.get_shape())
def testEmptyInput(self):
x = array_ops.ones(shape=[0, 3], dtype=dtypes.float32)
y = np.zeros(shape=[0, 3], dtype=np.float32)
self.assertEqual(0, self.evaluate(array_ops.size(x)))
self.assertAllEqual(y, self.evaluate(nn_ops.softmax(x, axis=0)))
def testDimTooLarge(self):
with self.cached_session():
# Use placeholder to make sure we get runtime error instead of shape
# inference error.
dim = array_ops.placeholder_with_default(100, shape=[])
with self.assertRaises(errors_impl.InvalidArgumentError):
nn_ops.softmax([1., 2., 3., 4.], axis=dim).eval()
def testInvalidAxis(self):
# Test case for GitHub issue 22793.
with self.cached_session():
ones = array_ops.ones(shape=[2, 3])
with self.assertRaises(errors_impl.InvalidArgumentError):
nn_ops.softmax(ones, axis=2).eval()
def testLargeDims(self):
# Make sure that we properly handle large inputs. See
# https://github.com/tensorflow/tensorflow/issues/4425 for details
for dims in [129, 256]:
ones = np.random.rand(dims, dims).astype(np.float32)
np_softmax = self._npSoftmax(ones)
for use_gpu in [True, False]:
with self.cached_session(use_gpu=use_gpu):
x = constant_op.constant(ones)
y = nn_ops.softmax(x)
tf_softmax = self.evaluate(y)
self.assertAllClose(tf_softmax, np_softmax)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/softplus_op_test.py
+148
View File
@@ -0,0 +1,148 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for Softplus and SoftplusGrad."""
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import gradients_impl
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
class SoftplusTest(test.TestCase):
def _npSoftplus(self, np_features):
np_features = np.asarray(np_features)
zero = np.asarray(0).astype(np_features.dtype)
return np.logaddexp(zero, np_features)
def _testSoftplus(self, np_features, use_gpu=False):
np_softplus = self._npSoftplus(np_features)
with self.cached_session(use_gpu=use_gpu):
softplus = nn_ops.softplus(np_features)
tf_softplus = self.evaluate(softplus)
self.assertAllCloseAccordingToType(
np_softplus, tf_softplus, half_rtol=5e-3, half_atol=5e-3,
bfloat16_rtol=5e-2, bfloat16_atol=5e-2
)
self.assertTrue(np.all(tf_softplus > 0))
self.assertShapeEqual(np_softplus, softplus)
def testNumbers(self):
for t in [
np.float16,
np.float32,
np.float64,
dtypes.bfloat16.as_numpy_dtype,
]:
self._testSoftplus(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
use_gpu=False)
self._testSoftplus(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
use_gpu=True)
if t == dtypes.bfloat16.as_numpy_dtype:
# bfloat16 dtype doesn't have finfo.
# Calculate epsilon using machine_epsilon = base ^ (-(precision - 1))
log_eps = np.log(2 ** (-(8 - 1)))
else:
log_eps = np.log(np.finfo(t).eps)
one = t(1)
ten = t(10)
self._testSoftplus(
[
log_eps, log_eps - one, log_eps + one, log_eps - ten,
log_eps + ten, -log_eps, -log_eps - one, -log_eps + one,
-log_eps - ten, -log_eps + ten
],
use_gpu=False)
self._testSoftplus(
[
log_eps, log_eps - one, log_eps + one, log_eps - ten,
log_eps + ten - log_eps, -log_eps - one, -log_eps + one,
-log_eps - ten, -log_eps + ten
],
use_gpu=True)
@test_util.run_deprecated_v1
def testGradient(self):
with self.cached_session():
x = constant_op.constant(
[-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
shape=[2, 5],
name="x")
y = nn_ops.softplus(x, name="softplus")
x_init = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float32,
order="F")
err = gradient_checker.compute_gradient_error(
x, [2, 5], y, [2, 5], x_init_value=x_init)
print("softplus (float) gradient err = ", err)
self.assertLess(err, 1e-4)
@test_util.run_deprecated_v1
def testGradGrad(self):
with self.cached_session():
x = constant_op.constant(
[-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
shape=[2, 5],
name="x")
y = nn_ops.softplus(x, name="softplus")
(grad,) = gradients_impl.gradients(y, x)
x_init = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float32,
order="F")
err = gradient_checker.compute_gradient_error(
x, [2, 5], grad, [2, 5], x_init_value=x_init)
print("softplus (float) gradient of gradient err = ", err)
self.assertLess(err, 5e-5)
@test_util.run_deprecated_v1
def testGradGradGrad(self):
with self.cached_session():
x = constant_op.constant(
[-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
shape=[2, 5],
name="x")
y = nn_ops.softplus(x, name="softplus")
(grad,) = gradients_impl.gradients(y, x)
(grad_grad,) = gradients_impl.gradients(grad, x)
x_init = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float32,
order="F")
err = gradient_checker.compute_gradient_error(
x, [2, 5], grad_grad, [2, 5], x_init_value=x_init)
print("softplus (float) third-order gradient err = ", err)
self.assertLess(err, 5e-5)
@test_util.run_deprecated_v1
def testNoInts(self):
with self.cached_session():
with self.assertRaisesRegex(
TypeError,
"'features' has DataType int32 not in list of allowed values"):
nn_ops.softplus(constant_op.constant(42)).eval()
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/softsign_op_test.py
+87
View File
@@ -0,0 +1,87 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for Softsign and SoftsignGrad."""
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad # pylint: disable=unused-import
from tensorflow.python.platform import test
class SoftsignTest(test.TestCase):
def _npSoftsign(self, np_features):
return np_features / (1 + np.abs(np_features))
def _testSoftsign(self, np_features, atol, rtol, use_gpu=False):
np_softsign = self._npSoftsign(np_features)
with self.cached_session(use_gpu=use_gpu):
softsign = nn_ops.softsign(np_features)
tf_softsign = self.evaluate(softsign)
self.assertAllClose(np_softsign, tf_softsign, rtol=rtol, atol=atol)
self.assertShapeEqual(np_softsign, softsign)
def testNumbers(self):
for t, atol, rtol in [
(np.float16, 1e-6, 1e-6),
(np.float32, 1e-6, 1e-6),
(np.float64, 1e-6, 1e-6),
(dtypes.bfloat16.as_numpy_dtype, 1e-2, 1e-2),
]:
self._testSoftsign(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
atol,
rtol,
use_gpu=False)
self._testSoftsign(
np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
atol,
rtol,
use_gpu=True)
@test_util.run_deprecated_v1
def testGradient(self):
with self.cached_session():
x = constant_op.constant(
[-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
shape=[2, 5],
name="x")
y = nn_ops.softsign(x, name="softsign")
x_init = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float32,
order="F")
err = gradient_checker.compute_gradient_error(
x, [2, 5], y, [2, 5], x_init_value=x_init)
print("softsign (float) gradient err = ", err)
self.assertLess(err, 1e-4)
@test_util.run_deprecated_v1
def testNoInts(self):
with self.cached_session():
with self.assertRaisesRegex(
TypeError,
"'features' has DataType int32 not in list of allowed values"):
nn_ops.softsign(constant_op.constant(7)).eval()
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/xent_op_d9m_test.py
+203
View File
@@ -0,0 +1,203 @@
# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for deterministic functionality of SoftmaxCrossEntropyWithLogits op."""
import numpy as np
from tensorflow.python.eager import backprop
from tensorflow.python.framework import config
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import test_util
from tensorflow.python.kernel_tests.nn_ops import xent_op_test_base
from tensorflow.python.ops import gen_nn_ops
from tensorflow.python.ops import nn_ops
# The following import is required to register the gradient function.
from tensorflow.python.ops.nn_grad import _SoftmaxCrossEntropyWithLogitsGrad # pylint: disable=unused-import
from tensorflow.python.platform import test
class XentOpDeterminismExceptionsTest(test.TestCase):
"""Test d9m-unimplemented exceptions from SoftmaxXentWithLogitsOp.
Test that tf.errors.UnimplementedError is thrown, as appropriate, by the GPU
code-paths through SoftmaxXentWithLogitsOp when deterministic ops are
enabled.
This test assumes that xent_op_test.py runs equivalent test cases when
deterministic ops are not enabled and will therefore detect erroneous
exception throwing in those cases.
"""
@test_util.run_gpu_only
@test_util.run_in_graph_and_eager_modes
def testExceptionThrowing(self):
with self.session(), test_util.force_gpu():
for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]:
features = constant_op.constant([[0.3, 0.5], [0.5, 0.6]], dtype=dtype)
labels = constant_op.constant([[0.2, 0.4], [0.1, 0.2]], dtype=dtype)
with self.assertRaisesRegex(
errors_impl.UnimplementedError,
"The GPU implementation of SoftmaxCrossEntropyWithLogits that " +
"would have been executed is not deterministic. Note that the " +
"Python API uses an alternative, deterministic, GPU-accelerated " +
"path when determinism is enabled."):
result = gen_nn_ops.softmax_cross_entropy_with_logits(
features=features, labels=labels)
self.evaluate(result)
class XentOpDeterministicTest(xent_op_test_base.XentOpTestBase):
"""Test that SoftmaxCrossEntropyWithLogits operates reproducibly.
Inheriting from xent_op_test_base.XentTestBase ensures that regular op
functionality is correct when the deterministic code-path is selected.
Note that because nn_ops.softmax_cross_entropy_with_logits calls
nn_ops.cross_entropy_with_logits_v2, the focus of testing is on the
former in order to test both.
"""
def _randomFloats(self, shape, dtype, normalized_rows=False):
a = (2 * np.random.random_sample(shape) - 1).astype(dtype)
if normalized_rows:
def normalize(row):
return row / row.sum()
a = np.apply_along_axis(normalize, 1, a)
return constant_op.constant(a)
def _generateInputs(self, dtype, seed=123, forward_not_backward=False):
batch_size = 1024
if forward_not_backward and dtype == np.float16:
# Generate more noise to expose the internal float32 implementation.
# This is associated with significantly slower test cases (esp. on CPU).
classes_count = 20000
else:
classes_count = 3000
shape = (batch_size, classes_count)
np.random.seed(seed)
labels = self._randomFloats(shape, dtype, normalized_rows=True)
logits = self._randomFloats(shape, dtype)
return labels, logits
@test_util.run_in_graph_and_eager_modes
def testForward(self):
with self.cached_session():
for dtype in [np.float16, np.float32, np.float64, \
dtypes.bfloat16.as_numpy_dtype]:
for trial in range(5):
seed = 123 + trial
labels, logits = self._generateInputs(
dtype, seed=seed, forward_not_backward=True)
result_a = nn_ops.softmax_cross_entropy_with_logits(
labels=labels, logits=logits)
result_b = nn_ops.softmax_cross_entropy_with_logits(
labels=labels, logits=logits)
self.assertAllEqual(result_a, result_b)
@test_util.run_in_graph_and_eager_modes
def testBackward(self):
with self.cached_session():
for dtype in [np.float16, np.float32, np.float64, \
dtypes.bfloat16.as_numpy_dtype]:
labels, logits = self._generateInputs(dtype, seed=456)
output_shape = labels.shape[0]
def gradients(seed):
np.random.seed(seed)
upstream_gradients = self._randomFloats(output_shape, dtype)
with backprop.GradientTape(persistent=True) as tape:
tape.watch(labels)
tape.watch(logits)
op_output = nn_ops.softmax_cross_entropy_with_logits(
labels=labels, logits=logits)
gradient_injector_output = op_output * upstream_gradients
return tape.gradient(gradient_injector_output, [labels, logits])
for trial in range(5):
seed = 456 + trial
labels_grad_a, logits_grad_a = gradients(seed=seed)
labels_grad_b, logits_grad_b = gradients(seed=seed)
self.assertAllEqual(labels_grad_a, labels_grad_b)
self.assertAllEqual(logits_grad_a, logits_grad_b)
# Modifications to the parent class (xent_op_test_base.XentOpTestBase) follow
def testSingleClass(self):
"""Modify testing of gradient for single-class case.
The deterministic implementation does not produce the gradients expected by
the original test (for the nondeterministic functionality) when the labels
vector is not a valid probability distribution.
labels: [[-1.], [0.], [1.], [1.]]
logits: [[1.], [-1.], [0.], [1.]]
nondeterministic deterministic
dloss/dlogits: [[2.0], [1.0], [0.0], [0.0]] [[0.0], [0.0], [0.0], [0.0]]
Note that only the second two label vectors are valid probability
distributions (as required by the API) and that the gradient matches for
those cases.
TODO(duncanriach): Further investigate the source of the difference in
the gradients for this case.
"""
self._testSingleClass(expected_gradient=[[0.0], [0.0], [0.0], [0.0]])
def testLabelsBroadcast(self):
"""Modify testing of gradient for labels-broadcast case.
The deterministic implementation does not produce the gradients expected by
the original test (for the nondeterministic functionality) when the labels
vector (after broadcasting) is not a valid probability distribution.
labels: [[0.], [2.], [0.25]]
logits: [[1., 1., 1., 1.],
[1., 2., 3., 4.],
[1., 2., 3., 4.]]
dloss/dlogits (nondeterministic):
[[ 0.25 , 0.25 , 0.25 , 0.25 ],
[-1.968, -1.913, -1.763, -1.355],
[-0.218, -0.163, -0.013, 0.394]]
dloss/dlogits (determinsitic):
[[ 0. , 0. , 0. , 0. ],
[-1.743, -1.303, -0.105, 3.150],
[-0.218, -0.163, -0.013, 0.394]]
Note that neither of the first two broadcast label vectors is a valid
probability distribution (as required by the API) and that these are the
cases that yield different gradients for nondeterministic vs determinsitic
implementations.
TODO(duncanriach): Further investigate the source of the difference in
the gradient for this case.
"""
self._testLabelsBroadcast(uniform_labels_gradient=[[
0., 0., 0., 0.
], [-1.743, -1.303, -0.105, 3.150], [-0.218, -0.163, -0.013, 0.394]])
if __name__ == "__main__":
config.enable_op_determinism()
test.main()
tensorflow/python/kernel_tests/nn_ops/xent_op_test.py
+133
View File
@@ -0,0 +1,133 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for SoftmaxCrossEntropyWithLogits op."""
import itertools
import sys
import numpy as np
from tensorflow.python.client import session
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.kernel_tests.nn_ops import xent_op_test_base
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_nn_ops
from tensorflow.python.ops import nn_ops
from tensorflow.python.platform import test
class XentOpTest(xent_op_test_base.XentOpTestBase):
@test_util.run_deprecated_v1
def testRankTooLarge(self):
for dtype in np.float16, np.float32:
np_features = np.array([[[1., 1., 1., 1.]], [[1., 2., 3.,
4.]]]).astype(dtype)
np_labels = np.array([[[0., 0., 0., 1.]], [[0., .5, .5,
0.]]]).astype(dtype)
self.assertRaisesRegex(ValueError, "rank 2, but is rank 3",
gen_nn_ops.softmax_cross_entropy_with_logits,
np_features, np_labels)
def testFeaturesBroadcast(self):
np_f = np.array([[1., 2., 3., 4.],
[1., 2., 3., 4.]]).astype(np.float32)
np_l = np.array([[0., 0., 0., 1.],
[0., .5, .5, 0.]]).astype(np.float32)
np_loss, np_gradient = self._npXent(labels=np_l, logits=np_f)
tf_f = constant_op.constant(
np.array([[1., 2., 3., 4.]]).astype(np.float32))
tf_l = constant_op.constant(
np.array([[0., 0., 0., 1.], [0., .5, .5, 0.]]).astype(np.float32))
tf_loss, tf_gradient = gen_nn_ops.softmax_cross_entropy_with_logits(
tf_f, tf_l)
self.assertAllCloseAccordingToType(np_loss, tf_loss)
self.assertAllCloseAccordingToType(np_gradient, tf_gradient)
tf_f = constant_op.constant(np.array([[1.]]).astype(np.float32))
tf_l = constant_op.constant(np.array([[1.], [1.]]).astype(np.float32))
tf_loss, tf_gradient = gen_nn_ops.softmax_cross_entropy_with_logits(
tf_f, tf_l)
self.assertAllClose([0, 0], tf_loss)
self.assertAllCloseAccordingToType([[0], [0]], tf_gradient)
@test_util.run_deprecated_v1
def testNotMatrix(self):
with self.cached_session():
with self.assertRaises(ValueError):
gen_nn_ops.softmax_cross_entropy_with_logits([0., 1., 2., 3.],
[0., 1., 0., 1.])
class XentBenchmark(test.Benchmark):
def benchmarkZeroDimension(self):
for (m, n, p, use_gpu) in itertools.product(
[128],
[10, 100, 1000, 10000, 100000],
[0.001, 0.01, 0.5, 0.99, 1.0],
[False]):
k = int(p * n)
if k == 0:
continue
name = "zero_dimension_m_%d_n_%d_k_%g_use_gpu_%s" % (m, n, k, use_gpu)
device = "/%s:0" % ("gpu" if use_gpu else "cpu")
with ops.Graph().as_default():
with ops.device(device):
labels = array_ops.zeros([0, 2, 4], dtype=dtypes.float32)
logits = array_ops.zeros([0, 2, 4], dtype=dtypes.float32)
op = nn_ops.softmax_cross_entropy_with_logits(
labels=labels, logits=logits)
with session.Session() as sess:
r = self.run_op_benchmark(sess, op, min_iters=100, name=name)
gb_processed_input = m * n / 1.0e9
throughput = gb_processed_input / r["wall_time"]
print("Benchmark: %s \t wall_time: %0.03g s \t "
"Throughput: %0.03g GB/s" % (name, r["wall_time"], throughput))
sys.stdout.flush()
def benchmarkSingleClass(self):
for (m, n, p, use_gpu) in itertools.product(
[128],
[10, 100, 1000, 10000, 100000],
[0.001, 0.01, 0.5, 0.99, 1.0],
[False]):
k = int(p * n)
if k == 0:
continue
name = "single_class_m_%d_n_%d_k_%g_use_gpu_%s" % (m, n, k, use_gpu)
device = "/%s:0" % ("gpu" if use_gpu else "cpu")
with ops.Graph().as_default():
with ops.device(device):
labels = constant_op.constant([[1.], [-1.], [0.]],
dtype=dtypes.float32)
logits = constant_op.constant([[-1.], [0.], [1.]],
dtype=dtypes.float32)
op = nn_ops.softmax_cross_entropy_with_logits(
labels=labels, logits=logits)
with session.Session() as sess:
r = self.run_op_benchmark(sess, op, min_iters=100, name=name)
gb_processed_input = m * n / 1.0e9
throughput = gb_processed_input / r["wall_time"]
print("Benchmark: %s \t wall_time: %0.03g s \t "
"Throughput: %0.03g GB/s" % (name, r["wall_time"], throughput))
sys.stdout.flush()
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/nn_ops/xent_op_test_base.py
+293
View File
@@ -0,0 +1,293 @@
# Copyright 2015-2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for SoftmaxCrossEntropyWithLogits op."""
import numpy as np
from tensorflow.python.eager import backprop
from tensorflow.python.framework import config
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import gradients_impl
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import nn_ops
# The following import is required to register the gradient function.
from tensorflow.python.ops.nn_grad import _SoftmaxCrossEntropyWithLogitsGrad # pylint: disable=unused-import
from tensorflow.python.platform import test
class XentOpTestBase(test.TestCase):
def _opFwdBwd(self, labels, logits, axis=-1):
""" Runs the op-under-test both forwards and backwards."""
logits = ops.convert_to_tensor(logits) # needed for the gradient tape
with backprop.GradientTape() as tape:
tape.watch(logits)
loss = nn_ops.softmax_cross_entropy_with_logits(
labels=labels, logits=logits, dim=axis)
return loss, tape.gradient(loss, logits)
def _npXent(self, labels, logits, dim=-1):
if dim == -1:
dim = len(logits.shape) - 1
one_only_on_dim = list(logits.shape)
one_only_on_dim[dim] = 1
e = np.exp(logits - np.reshape(np.amax(logits, axis=dim), one_only_on_dim))
probs = e / np.reshape(np.sum(e, axis=dim), one_only_on_dim)
bp = (probs - labels)
l = -np.sum(labels * np.log(probs + 1.0e-20), axis=dim)
return l, bp
# TODO(b/123860949): The values are constant folded for XLA, so placeholders
# are needed.
def _testXent2D(self,
np_labels,
np_logits,
with_placeholders=False,
expected_gradient=None):
np_loss, np_gradient = self._npXent(labels=np_labels, logits=np_logits)
if expected_gradient is not None:
np_gradient = expected_gradient
with self.cached_session() as sess:
if with_placeholders:
logits_placeholder = array_ops.placeholder(np_logits.dtype)
labels_placeholder = array_ops.placeholder(np_labels.dtype)
loss, gradient = self._opFwdBwd(labels_placeholder, logits_placeholder)
tf_loss, tf_gradient = sess.run([loss, gradient],
feed_dict={
labels_placeholder: np_labels,
logits_placeholder: np_logits
})
else:
loss, gradient = self._opFwdBwd(np_labels, np_logits)
tf_loss, tf_gradient = self.evaluate([loss, gradient])
self.assertAllCloseAccordingToType(np_loss, tf_loss, half_rtol=1e-2)
self.assertAllCloseAccordingToType(np_gradient, tf_gradient)
def _testXentND(self, np_labels, np_logits, dim=-1):
np_loss, _ = self._npXent(np_labels, np_logits, dim=dim)
loss = nn_ops.softmax_cross_entropy_with_logits(
labels=np_labels, logits=np_logits, dim=dim)
tf_loss = self.evaluate(loss)
self.assertAllCloseAccordingToType(np_loss, tf_loss)
def _testSingleClass(self, expected_gradient=[[2.0], [1.0], [0.0], [0.0]]):
for dtype in np.float16, np.float32, dtypes.bfloat16.as_numpy_dtype:
loss, gradient = self._opFwdBwd(
labels=np.array([[-1.], [0.], [1.], [1.]]).astype(dtype),
logits=np.array([[1.], [-1.], [0.], [1.]]).astype(dtype))
self.assertAllClose([0.0, 0.0, 0.0, 0.0], loss)
self.assertAllClose(expected_gradient, gradient)
def testSingleClass(self):
"""This method is structured to be easily overridden by a child class."""
self._testSingleClass()
def testNpXent(self):
# We create 2 batches of logits for testing.
# batch 0 is the boring uniform distribution: 1, 1, 1, 1, with target 3.
# batch 1 has a bit of difference: 1, 2, 3, 4, with soft targets (1, 2).
logits = [[1., 1., 1., 1.], [1., 2., 3., 4.]]
labels = [[0., 0., 0., 1.], [0., .5, .5, 0.]]
# For batch 0, we expect the uniform distribution: 0.25, 0.25, 0.25, 0.25
# With a hard target 3, the gradient is [0.25, 0.25, 0.25, -0.75]
# The loss for this batch is -log(0.25) = 1.386
#
# For batch 1, we have:
# exp(0) = 1
# exp(1) = 2.718
# exp(2) = 7.389
# exp(3) = 20.085
# SUM = 31.192
# So we have as probabilities:
# exp(0) / SUM = 0.032
# exp(1) / SUM = 0.087
# exp(2) / SUM = 0.237
# exp(3) / SUM = 0.644
# With a soft target (1, 2), the gradient is
# [0.032, 0.087 - 0.5 = -0.413, 0.237 - 0.5 = -0.263, 0.644]
# The loss for this batch is [0.5 * -log(0.087), 0.5 * -log(0.237)]
# = [1.3862, 1.9401]
np_loss, np_gradient = self._npXent(np.array(labels), np.array(logits))
self.assertAllClose(
np.array([[0.25, 0.25, 0.25, -0.75], [0.0321, -0.4129, -0.2632,
0.6439]]),
np_gradient,
rtol=1.e-3,
atol=1.e-3)
self.assertAllClose(
np.array([1.3862, 1.9401]), np_loss, rtol=1.e-3, atol=1.e-3)
# TODO(b/123860949): The values are constant folded for XLA, so placeholders
# are needed.
@test_util.run_deprecated_v1
def _testLabelsBroadcast(self, uniform_labels_gradient):
labels = np.array([[0., 0., 0., 1.]]).astype(np.float16)
logits = np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float16)
self._testXent2D(labels, logits, with_placeholders=True)
labels = np.array([[1.]]).astype(np.float16)
logits = np.array([[1.], [2.]]).astype(np.float16)
self._testXent2D(labels, logits, with_placeholders=True)
labels = np.array([[0.], [2.], [0.25]]).astype(np.float16)
logits = np.array([[1., 1., 1., 1.], [1., 2., 3., 4.],
[1., 2., 3., 4.]]).astype(np.float16)
self._testXent2D(
labels,
logits,
with_placeholders=True,
expected_gradient=uniform_labels_gradient)
def testLabelsBroadcast(self):
"""This method is structured to be easily overridden by a child class."""
self._testLabelsBroadcast(uniform_labels_gradient=[[
0.25, 0.25, 0.25, 0.25
], [-1.968, -1.913, -1.763, -1.355], [-0.218, -0.163, -0.013, 0.394]])
@test_util.run_deprecated_v1
def testShapeMismatch(self):
with self.cached_session():
with self.assertRaises(ValueError):
self._opFwdBwd(
labels=[[0., 1., 0.], [1., 0., 0.]], logits=[[0., 1.], [2., 3.]])
def testHalf(self):
labels = np.array([[0., 0., 0., 1.], [0., .5, .5, 0.]]).astype(np.float16)
logits = np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float16)
self._testXent2D(labels, logits)
def testBfloat16(self):
labels = np.array([[0., 0., 0., 1.],
[0., .5, .5, 0.]]).astype(dtypes.bfloat16.as_numpy_dtype)
logits = np.array([[1., 1., 1., 1.],
[1., 2., 3., 4.]]).astype(dtypes.bfloat16.as_numpy_dtype)
self._testXent2D(labels, logits)
def testFloat(self):
labels = np.array([[0., 0., 0., 1.], [0., .5, .5, 0.]]).astype(np.float32)
logits = np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float32)
self._testXent2D(labels, logits)
def testDouble(self):
labels = np.array([[0., 0., 0., 1.], [0., .5, .5, 0.]]).astype(np.float64)
logits = np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float64)
self._testXent2D(labels, logits)
@test_util.run_deprecated_v1
def testGradient(self):
with self.cached_session() as sess:
labels = constant_op.constant(
[0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.5, 0.0, 0.5],
shape=[3, 4],
dtype=dtypes.float64,
name="labels")
logits = constant_op.constant(
[0.1, 0.2, 0.3, 0.4, 0.1, 0.4, 0.9, 1.6, 0.1, 0.8, 2.7, 6.4],
shape=[3, 4],
dtype=dtypes.float64,
name="logits")
x = nn_ops.softmax_cross_entropy_with_logits(
labels=labels, logits=logits, name="xent")
err = gradient_checker.compute_gradient_error(logits, [3, 4], x, [3])
# Check that no extra computation gets performed. When only the first
# derivative is requested, the second derivative must not be computed.
# So when there is no second derivative, there is no `BatchMatMul` op
# in the graph.
op_names = [
op.op_def.name for op in sess.graph.get_operations() if op.op_def
]
self.assertNotIn("BatchMatMul", op_names)
self.assertNotIn("BatchMatMulV2", op_names)
self.assertLess(err, 5e-8)
@test_util.run_deprecated_v1
def testGradientLabelWithV2(self):
with self.cached_session():
labels = constant_op.constant(
[0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.5, 0.0, 0.5],
shape=[3, 4],
dtype=dtypes.float64,
name="labels")
logits = constant_op.constant(
[0.1, 0.2, 0.3, 0.4, 0.1, 0.4, 0.9, 1.6, 0.1, 0.8, 2.7, 6.4],
shape=[3, 4],
dtype=dtypes.float64,
name="logits")
x = nn_ops.softmax_cross_entropy_with_logits_v2(
labels=labels, logits=logits, name="xent")
err = gradient_checker.compute_gradient_error(labels, [3, 4], x, [3])
self.assertLess(err, 5e-8)
@test_util.run_deprecated_v1
def testSecondGradient(self):
with self.cached_session() as sess:
labels = constant_op.constant([
0.0, 0.0, 1.0 / 3, 0.0, 1.0 / 3, 0.0, 0.0, 0.0, 0.0, 0.5 / 3, 0.0,
0.5 / 3
],
shape=[12],
dtype=dtypes.float64,
name="labels")
logits = constant_op.constant(
[0.1, 0.2, 0.3, 0.4, 0.1, 0.4, 0.9, 1.6, 0.1, 0.8, 2.7, 6.4],
shape=[12],
dtype=dtypes.float64,
name="logits")
x = nn_ops.softmax_cross_entropy_with_logits(
labels=labels, logits=logits, name="xent")
loss = math_ops.reduce_sum(x)
gradients = gradients_impl.gradients(loss, [logits])[0]
err = gradient_checker.compute_gradient_error(logits, [12], gradients,
[12])
if not config.is_op_determinism_enabled():
# Check how second derivative is calculated.
# (it is equivalent to a `BatchMatMul` op being in the graph because of
# the implementation in SoftmaxCrossEntropyWithLogitsGrad)
op_names = [
op.op_def.name for op in sess.graph.get_operations() if op.op_def
]
self.assertIn("BatchMatMulV2", op_names)
self.assertLess(err, 5e-8)
def test3D(self):
labels = np.array([[[0., 0., 0., 1.], [0., 1., 0., 0.]],
[[0., 0.5, 0.5, 0.], [0.5, 0.5, 0., 0.]],
[[0., 1., 0., 0.], [0., 0., 1., 0.]]]).astype(np.float32)
logits = np.array([[[1., 1., 1., 1.], [1., 2., 3., 4.]],
[[2., 3., 4., 5.], [6., 7., 8., 9.]],
[[5., 4., 3., 2.], [1., 2., 3., 4.]]]).astype(np.float32)
self._testXentND(labels, logits, dim=0)
self._testXentND(labels, logits, dim=1)
self._testXentND(labels, logits, dim=-1)
def testZeroDimension(self):
labels = np.zeros([0, 2, 4]).astype(np.float32)
logits = np.zeros([0, 2, 4]).astype(np.float32)
np_loss, _ = self._npXent(labels=labels, logits=logits)
loss = nn_ops.softmax_cross_entropy_with_logits(
labels=labels, logits=logits)
tf_loss = self.evaluate(loss)
self.assertAllEqual(np_loss, tf_loss)