wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions 3 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/python/kernel_tests/strings_ops/BUILD
+313
View File
@@ -0,0 +1,313 @@
# Tests of TensorFlow math kernels written using the Python API.
load("//tensorflow:tensorflow.default.bzl", "cuda_py_strict_test", "tf_py_strict_test")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
licenses = ["notice"],
)
tf_py_strict_test(
name = "as_string_op_test",
size = "small",
srcs = ["as_string_op_test.py"],
tags = ["no_windows"],
deps = [
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
tf_py_strict_test(
name = "base64_ops_test",
size = "small",
srcs = ["base64_ops_test.py"],
tags = ["nomac"], # b/35468214
deps = [
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "reduce_join_op_test",
size = "small",
srcs = ["reduce_join_op_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
tf_py_strict_test(
name = "regex_full_match_op_test",
size = "small",
srcs = ["regex_full_match_op_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/ops:string_ops_gen",
"//tensorflow/python/platform:client_testlib",
"@absl_py//absl/testing:parameterized",
],
)
tf_py_strict_test(
name = "regex_replace_op_test",
size = "small",
srcs = ["regex_replace_op_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/ops:string_ops_gen",
"//tensorflow/python/platform:client_testlib",
"@absl_py//absl/testing:parameterized",
],
)
tf_py_strict_test(
name = "string_bytes_split_op_test",
size = "small",
srcs = ["string_bytes_split_op_test.py"],
deps = [
"//tensorflow/python/eager:def_function",
"//tensorflow/python/framework:tensor_spec",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops/ragged:ragged_factory_ops",
"//tensorflow/python/ops/ragged:ragged_string_ops",
"//tensorflow/python/platform:client_testlib",
"@absl_py//absl/testing:parameterized",
],
)
tf_py_strict_test(
name = "string_format_op_test",
size = "small",
srcs = ["string_format_op_test.py"],
deps = [
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/util:compat",
],
)
tf_py_strict_test(
name = "string_join_op_test",
size = "small",
srcs = ["string_join_op_test.py"],
deps = [
"//tensorflow/python/framework:errors",
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/platform:client_testlib",
],
)
tf_py_strict_test(
name = "string_length_op_test",
size = "small",
srcs = ["string_length_op_test.py"],
deps = [
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/platform:client_testlib",
],
)
tf_py_strict_test(
name = "string_lower_op_test",
size = "small",
srcs = ["string_lower_op_test.py"],
deps = [
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/platform:client_testlib",
],
)
tf_py_strict_test(
name = "string_split_op_test",
size = "small",
srcs = ["string_split_op_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:sparse_tensor",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/ops/ragged:ragged_factory_ops",
"//tensorflow/python/ops/ragged:ragged_string_ops",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/util:compat",
"//third_party/py/numpy",
"@absl_py//absl/testing:parameterized",
],
)
tf_py_strict_test(
name = "string_strip_op_test",
size = "small",
srcs = ["string_strip_op_test.py"],
deps = [
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/platform:client_testlib",
],
)
cuda_py_strict_test(
name = "string_to_hash_bucket_op_test",
size = "small",
srcs = ["string_to_hash_bucket_op_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/platform:client_testlib",
],
)
cuda_py_strict_test(
name = "string_to_number_op_test",
size = "small",
srcs = ["string_to_number_op_test.py"],
deps = [
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:parsing_ops",
"//tensorflow/python/platform:client_testlib",
],
)
tf_py_strict_test(
name = "string_upper_op_test",
size = "small",
srcs = ["string_upper_op_test.py"],
deps = [
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/platform:client_testlib",
],
)
tf_py_strict_test(
name = "substr_op_test",
size = "small",
srcs = ["substr_op_test.py"],
deps = [
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
"@absl_py//absl/testing:parameterized",
],
)
tf_py_strict_test(
name = "unicode_decode_op_test",
size = "small",
srcs = ["unicode_decode_op_test.py"],
deps = [
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:sparse_tensor",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:string_ops_gen",
"//tensorflow/python/ops/ragged:ragged_factory_ops",
"//tensorflow/python/ops/ragged:ragged_string_ops",
"//tensorflow/python/platform:client_testlib",
"@absl_py//absl/testing:parameterized",
],
)
tf_py_strict_test(
name = "unicode_encode_op_test",
size = "small",
srcs = ["unicode_encode_op_test.py"],
deps = [
"//tensorflow/python/eager:def_function",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:tensor_spec",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops/ragged:ragged_factory_ops",
"//tensorflow/python/ops/ragged:ragged_string_ops",
"//tensorflow/python/ops/ragged:ragged_tensor",
"//tensorflow/python/ops/ragged:ragged_tensor_value",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
"@absl_py//absl/testing:parameterized",
],
)
tf_py_strict_test(
name = "unicode_script_op_test",
size = "small",
srcs = ["unicode_script_op_test.py"],
deps = [
"//tensorflow/python/client:session",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/platform:benchmark",
"//tensorflow/python/platform:client_testlib",
],
)
tf_py_strict_test(
name = "unicode_transcode_op_test",
size = "small",
srcs = ["unicode_transcode_op_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/platform:client_testlib",
"@absl_py//absl/testing:parameterized",
],
)
cuda_py_strict_test(
name = "unsorted_segment_join_op_test",
size = "small",
srcs = ["unsorted_segment_join_op_test.py"],
deps = [
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:string_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
"@absl_py//absl/testing:parameterized",
],
)
tensorflow/python/kernel_tests/strings_ops/as_string_op_test.py
+209
View File
@@ -0,0 +1,209 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for as_string_op."""
import numpy as np
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.ops import string_ops
from tensorflow.python.platform import test
class AsStringOpTest(test.TestCase):
def testFloat(self):
float_inputs_ = [
0, 1, -1, 0.5, 0.25, 0.125, float("INF"), float("NAN"), float("-INF")
]
for dtype in (dtypes.half, dtypes.bfloat16, dtypes.float32, dtypes.float64):
inputs = ops.convert_to_tensor(float_inputs_, dtype=dtype)
s = lambda strs: [x.decode("ascii") for x in self.evaluate(strs)]
result = string_ops.as_string(inputs, shortest=True)
self.assertAllEqual(s(result), ["%g" % x for x in float_inputs_])
result = string_ops.as_string(inputs, scientific=True)
self.assertAllEqual(s(result), ["%e" % x for x in float_inputs_])
result = string_ops.as_string(inputs)
self.assertAllEqual(s(result), ["%f" % x for x in float_inputs_])
result = string_ops.as_string(inputs, width=3)
self.assertAllEqual(s(result), ["%3f" % x for x in float_inputs_])
result = string_ops.as_string(inputs, width=3, fill="0")
self.assertAllEqual(s(result), ["%03f" % x for x in float_inputs_])
result = string_ops.as_string(inputs, width=3, fill="0", shortest=True)
self.assertAllEqual(s(result), ["%03g" % x for x in float_inputs_])
result = string_ops.as_string(inputs, precision=10, width=3)
self.assertAllEqual(s(result), ["%03.10f" % x for x in float_inputs_])
result = string_ops.as_string(
inputs, precision=10, width=3, fill="0", shortest=True
)
self.assertAllEqual(s(result), ["%03.10g" % x for x in float_inputs_])
with self.assertRaisesOpError("Cannot select both"):
self.evaluate(
string_ops.as_string(inputs, scientific=True, shortest=True)
)
with self.assertRaisesOpError("Fill string must be one or fewer"):
self.evaluate(string_ops.as_string(inputs, fill="ab"))
def testInt(self):
# Cannot use values outside -128..127 for test, because we're also
# testing int8
int_inputs = [0, -1, 1, -128, 127, -101, 101, -0]
int_dtypes = [dtypes.int8, dtypes.int32, dtypes.int64]
uint_inputs = [0, 1, 127, 255, 101]
uint_dtypes = [dtypes.uint8, dtypes.uint32, dtypes.uint64]
s = lambda strs: [x.decode("ascii") for x in self.evaluate(strs)]
for dtypes_, inputs_ in [
(int_dtypes, int_inputs),
(uint_dtypes, uint_inputs),
]:
for dtype in dtypes_:
inputs = ops.convert_to_tensor(inputs_, dtype=dtype)
result = string_ops.as_string(inputs)
self.assertAllEqual(s(result), ["%d" % x for x in inputs_])
result = string_ops.as_string(inputs, width=3)
self.assertAllEqual(s(result), ["%3d" % x for x in inputs_])
result = string_ops.as_string(inputs, width=3, fill="0")
self.assertAllEqual(s(result), ["%03d" % x for x in inputs_])
with self.assertRaisesOpError("scientific and shortest"):
self.evaluate(string_ops.as_string(inputs, scientific=True))
with self.assertRaisesOpError("scientific and shortest"):
self.evaluate(string_ops.as_string(inputs, shortest=True))
with self.assertRaisesOpError("precision not supported"):
self.evaluate(string_ops.as_string(inputs, precision=0))
def testLargeInt(self):
# Cannot use values outside -128..127 for test, because we're also
# testing int8
s = lambda strs: [x.decode("ascii") for x in self.evaluate(strs)]
inputs = [np.iinfo(np.int32).min, np.iinfo(np.int32).max]
result = string_ops.as_string(inputs)
self.assertAllEqual(s(result), ["%d" % x for x in inputs])
inputs = [np.iinfo(np.int64).min, np.iinfo(np.int64).max]
result = string_ops.as_string(inputs)
self.assertAllEqual(s(result), ["%d" % x for x in inputs])
def testHalfInt(self):
s = lambda strs: [x.decode("ascii") for x in self.evaluate(strs)]
for dtype, np_dtype in [
(dtypes.int16, np.int16),
(dtypes.uint16, np.uint16),
]:
inputs = [np.iinfo(np_dtype).min, np.iinfo(np_dtype).max]
result = string_ops.as_string(ops.convert_to_tensor(inputs, dtype=dtype))
self.assertAllEqual(s(result), ["%d" % x for x in inputs])
def testBool(self):
bool_inputs_ = [False, True]
s = lambda strs: [x.decode("ascii") for x in self.evaluate(strs)]
result = string_ops.as_string(bool_inputs_)
self.assertAllEqual(s(result), ["false", "true"])
def testComplex(self):
inputs = [
0,
1,
-1,
0.5,
0.25,
0.125,
complex("INF"),
complex("NAN"),
complex("-INF"),
]
complex_inputs_ = [(x + (x + 1) * 1j) for x in inputs]
for dtype in (dtypes.complex64, dtypes.complex128):
inputs = ops.convert_to_tensor(complex_inputs_, dtype=dtype)
def clean_nans(s_l):
return [
s.decode("ascii").replace("-nan", "nan") for s in self.evaluate(s_l)
]
result = string_ops.as_string(inputs, shortest=True)
self.assertAllEqual(
clean_nans(result),
["(%g,%g)" % (x.real, x.imag) for x in complex_inputs_],
)
result = string_ops.as_string(inputs, scientific=True)
self.assertAllEqual(
clean_nans(result),
["(%e,%e)" % (x.real, x.imag) for x in complex_inputs_],
)
result = string_ops.as_string(inputs)
self.assertAllEqual(
clean_nans(result),
["(%f,%f)" % (x.real, x.imag) for x in complex_inputs_],
)
result = string_ops.as_string(inputs, width=3)
self.assertAllEqual(
clean_nans(result),
["(%03f,%03f)" % (x.real, x.imag) for x in complex_inputs_],
)
result = string_ops.as_string(inputs, width=3, fill="0", shortest=True)
self.assertAllEqual(
clean_nans(result),
["(%03g,%03g)" % (x.real, x.imag) for x in complex_inputs_],
)
result = string_ops.as_string(inputs, precision=10, width=3)
self.assertAllEqual(
clean_nans(result),
["(%03.10f,%03.10f)" % (x.real, x.imag) for x in complex_inputs_],
)
result = string_ops.as_string(
inputs, precision=10, width=3, fill="0", shortest=True
)
self.assertAllEqual(
clean_nans(result),
["(%03.10g,%03.10g)" % (x.real, x.imag) for x in complex_inputs_],
)
with self.assertRaisesOpError("Cannot select both"):
self.evaluate(
string_ops.as_string(inputs, scientific=True, shortest=True)
)
def testString(self):
self.assertAllEqual(string_ops.as_string("hello, world!"), "hello, world!")
widened_string = self.evaluate(
string_ops.as_string("hello, world!", width=20)
)
self.assertAllEqual(widened_string, " hello, world!")
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/base64_ops_test.py
+152
View File
@@ -0,0 +1,152 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for EncodeBase64 and DecodeBase64."""
import base64
import numpy as np
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import string_ops
from tensorflow.python.platform import test
@test_util.run_deprecated_v1
class Base64OpsTest(test_util.TensorFlowTestCase):
def setUp(self):
self._msg = array_ops.placeholder(dtype=dtypes.string)
self._encoded_f = string_ops.encode_base64(self._msg, pad=False)
self._decoded_f = string_ops.decode_base64(self._encoded_f)
self._encoded_t = string_ops.encode_base64(self._msg, pad=True)
self._decoded_t = string_ops.decode_base64(self._encoded_t)
def _RemovePad(self, msg, base64_msg):
if len(msg) % 3 == 1:
return base64_msg[:-2]
if len(msg) % 3 == 2:
return base64_msg[:-1]
return base64_msg
def _RunTest(self, msg, pad):
with self.cached_session() as sess:
if pad:
encoded, decoded = sess.run([self._encoded_t, self._decoded_t],
feed_dict={self._msg: msg})
else:
encoded, decoded = sess.run([self._encoded_f, self._decoded_f],
feed_dict={self._msg: msg})
if not isinstance(msg, (list, tuple)):
msg = [msg]
encoded = [encoded]
decoded = [decoded]
base64_msg = [base64.urlsafe_b64encode(m) for m in msg]
if not pad:
base64_msg = [self._RemovePad(m, b) for m, b in zip(msg, base64_msg)]
for i in range(len(msg)):
self.assertEqual(base64_msg[i], encoded[i])
self.assertEqual(msg[i], decoded[i])
def testWithPythonBase64(self):
for pad in (False, True):
self._RunTest(b"", pad=pad)
for _ in range(100):
length = np.random.randint(1024 * 1024)
msg = np.random.bytes(length)
self._RunTest(msg, pad=pad)
def testShape(self):
for pad in (False, True):
for _ in range(10):
msg = [np.random.bytes(np.random.randint(20))
for _ in range(np.random.randint(10))]
self._RunTest(msg, pad=pad)
# Zero-element, non-trivial shapes.
for _ in range(10):
k = np.random.randint(10)
msg = np.empty((0, k), dtype=bytes)
encoded = string_ops.encode_base64(msg, pad=pad)
decoded = string_ops.decode_base64(encoded)
with self.cached_session() as sess:
encoded_value, decoded_value = self.evaluate([encoded, decoded])
self.assertEqual(encoded_value.shape, msg.shape)
self.assertEqual(decoded_value.shape, msg.shape)
def testInvalidInput(self):
def try_decode(enc):
self._decoded_f.eval(feed_dict={self._encoded_f: enc})
with self.cached_session():
# Invalid length.
msg = np.random.bytes(99)
enc = base64.urlsafe_b64encode(msg)
with self.assertRaisesRegex(errors.InvalidArgumentError, "1 modulo 4"):
try_decode(enc + b"a")
# Invalid char used in encoding.
msg = np.random.bytes(34)
enc = base64.urlsafe_b64encode(msg)
for i in range(len(msg)):
with self.assertRaises(errors.InvalidArgumentError):
try_decode(enc[:i] + b"?" + enc[(i + 1):])
with self.assertRaises(errors.InvalidArgumentError):
try_decode(enc[:i] + b"\x80" + enc[(i + 1):]) # outside ascii range.
with self.assertRaises(errors.InvalidArgumentError):
try_decode(enc[:i] + b"+" + enc[(i + 1):]) # not url-safe.
with self.assertRaises(errors.InvalidArgumentError):
try_decode(enc[:i] + b"/" + enc[(i + 1):]) # not url-safe.
# Partial padding.
msg = np.random.bytes(34)
enc = base64.urlsafe_b64encode(msg)
with self.assertRaises(errors.InvalidArgumentError):
# enc contains == at the end. Partial padding is not allowed.
try_decode(enc[:-1])
# Unnecessary padding.
msg = np.random.bytes(33)
enc = base64.urlsafe_b64encode(msg)
with self.assertRaises(errors.InvalidArgumentError):
try_decode(enc + b"==")
with self.assertRaises(errors.InvalidArgumentError):
try_decode(enc + b"===")
with self.assertRaises(errors.InvalidArgumentError):
try_decode(enc + b"====")
# Padding in the middle. (Previous implementation was ok with this as long
# as padding char location was 2 or 3 (mod 4).
msg = np.random.bytes(33)
enc = base64.urlsafe_b64encode(msg)
for i in range(len(msg) - 1):
with self.assertRaises(errors.InvalidArgumentError):
try_decode(enc[:i] + b"=" + enc[(i + 1):])
for i in range(len(msg) - 2):
with self.assertRaises(errors.InvalidArgumentError):
try_decode(enc[:i] + b"==" + enc[(i + 2):])
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/reduce_join_op_test.py
+360
View File
@@ -0,0 +1,360 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for ReduceJoin op from string_ops."""
import itertools
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import string_ops
from tensorflow.python.platform import test
def _input_array(num_dims):
"""Creates an ndarray where each element is the binary of its linear index.
Args:
num_dims: The number of dimensions to create.
Returns:
An ndarray of shape [2] * num_dims.
"""
formatter = "{:0%db}" % num_dims
strings = [formatter.format(i) for i in range(2**num_dims)]
return np.array(strings, dtype="S%d" % num_dims).reshape([2] * num_dims)
def _joined_array(num_dims, reduce_dim):
"""Creates an ndarray with the result from reduce_join on input_array.
Args:
num_dims: The number of dimensions of the original input array.
reduce_dim: The dimension to reduce.
Returns:
An ndarray of shape [2] * (num_dims - 1).
"""
formatter = "{:0%db}" % (num_dims - 1)
result = np.zeros(shape=[2] * (num_dims - 1), dtype="S%d" % (2 * num_dims))
flat = result.ravel()
for i in range(2**(num_dims - 1)):
dims = formatter.format(i)
flat[i] = "".join([
(dims[:reduce_dim] + "%d" + dims[reduce_dim:]) % j for j in range(2)
])
return result
class UnicodeTestCase(test.TestCase):
"""Test case with Python3-compatible string comparator."""
def assertAllEqualUnicode(self, truth, actual):
self.assertAllEqual(
np.array(truth).astype("U"), np.array(actual).astype("U"))
class ReduceJoinTestHelperTest(UnicodeTestCase):
"""Tests for helper functions."""
def testInputArray(self):
num_dims = 3
truth = ["{:03b}".format(i) for i in range(2**num_dims)]
output_array = _input_array(num_dims).reshape([-1])
self.assertAllEqualUnicode(truth, output_array)
def testJoinedArray(self):
num_dims = 3
truth_dim_zero = [["000100", "001101"], ["010110", "011111"]]
truth_dim_one = [["000010", "001011"], ["100110", "101111"]]
truth_dim_two = [["000001", "010011"], ["100101", "110111"]]
output_array_dim_zero = _joined_array(num_dims, reduce_dim=0)
output_array_dim_one = _joined_array(num_dims, reduce_dim=1)
output_array_dim_two = _joined_array(num_dims, reduce_dim=2)
self.assertAllEqualUnicode(truth_dim_zero, output_array_dim_zero)
self.assertAllEqualUnicode(truth_dim_one, output_array_dim_one)
self.assertAllEqualUnicode(truth_dim_two, output_array_dim_two)
class ReduceJoinTest(UnicodeTestCase):
def _testReduceJoin(self,
input_array,
truth,
truth_shape,
axis,
keep_dims=False,
separator=""):
"""Compares the output of reduce_join to an expected result.
Args:
input_array: The string input to be joined.
truth: An array or np.array of the expected result.
truth_shape: An array or np.array of the expected shape.
axis: The indices to reduce over.
keep_dims: Whether or not to retain reduced dimensions.
separator: The separator to use for joining.
"""
with self.cached_session():
output = string_ops.reduce_join(
inputs=input_array,
axis=axis,
keep_dims=keep_dims,
separator=separator)
output_array = self.evaluate(output)
self.assertAllEqualUnicode(truth, output_array)
self.assertAllEqual(truth_shape, output.get_shape())
def _testMultipleReduceJoin(self, input_array, axis, separator=" "):
"""Tests reduce_join for one input and multiple axes.
Does so by comparing the output to that from nested reduce_string_joins.
The correctness of single-dimension reduce_join is verified by other
tests below using _testReduceJoin.
Args:
input_array: The input to test.
axis: The indices to reduce.
separator: The separator to use when joining.
"""
with self.cached_session():
output = string_ops.reduce_join(
inputs=input_array, axis=axis, keep_dims=False, separator=separator)
output_keep_dims = string_ops.reduce_join(
inputs=input_array, axis=axis, keep_dims=True, separator=separator)
truth = input_array
for index in axis:
truth = string_ops.reduce_join(
inputs=truth, axis=index, keep_dims=True, separator=separator)
if not axis:
truth = constant_op.constant(truth)
truth_squeezed = array_ops.squeeze(truth, axis=axis)
output_array = self.evaluate(output)
output_keep_dims_array = self.evaluate(output_keep_dims)
truth_array = self.evaluate(truth)
truth_squeezed_array = self.evaluate(truth_squeezed)
self.assertAllEqualUnicode(truth_array, output_keep_dims_array)
self.assertAllEqualUnicode(truth_squeezed_array, output_array)
self.assertAllEqual(truth.get_shape(), output_keep_dims.get_shape())
self.assertAllEqual(truth_squeezed.get_shape(), output.get_shape())
def testRankOne(self):
input_array = ["this", "is", "a", "test"]
truth = "thisisatest"
truth_shape = []
self._testReduceJoin(input_array, truth, truth_shape, axis=0)
def testRankTwo(self):
input_array = [["this", "is", "a", "test"],
["please", "do", "not", "panic"]]
truth_dim_zero = ["thisplease", "isdo", "anot", "testpanic"]
truth_shape_dim_zero = [4]
truth_dim_one = ["thisisatest", "pleasedonotpanic"]
truth_shape_dim_one = [2]
self._testReduceJoin(
input_array, truth_dim_zero, truth_shape_dim_zero, axis=0)
self._testReduceJoin(
input_array, truth_dim_one, truth_shape_dim_one, axis=1)
expected_val = "thisisatestpleasedonotpanic"
expected_shape = []
self._testReduceJoin(input_array, expected_val, expected_shape, axis=None)
# Using axis=[] is a no-op.
expected_val = input_array
expected_shape = [2, 4]
self._testReduceJoin(input_array, expected_val, expected_shape, axis=[])
def testRankFive(self):
input_array = _input_array(num_dims=5)
truths = [_joined_array(num_dims=5, reduce_dim=i) for i in range(5)]
truth_shape = [2] * 4
for i in range(5):
self._testReduceJoin(input_array, truths[i], truth_shape, axis=i)
def testNegative(self):
input_array = _input_array(num_dims=5)
truths = [_joined_array(num_dims=5, reduce_dim=i) for i in range(5)]
truth_shape = [2] * 4
for i in range(5):
self._testReduceJoin(input_array, truths[i], truth_shape, axis=i - 5)
def testSingletonDimension(self):
input_arrays = [
_input_array(num_dims=5).reshape([2] * i + [1] + [2] * (5 - i))
for i in range(6)
]
truth = _input_array(num_dims=5)
truth_shape = [2] * 5
for i in range(6):
self._testReduceJoin(input_arrays[i], truth, truth_shape, axis=i)
def testSeparator(self):
input_array = [["this", "is", "a", "test"],
["please", "do", "not", "panic"]]
truth_dim_zero = ["this please", "is do", "a not", "test panic"]
truth_shape_dim_zero = [4]
truth_dim_one = ["this is a test", "please do not panic"]
truth_shape_dim_one = [2]
self._testReduceJoin(
input_array,
truth_dim_zero,
truth_shape_dim_zero,
axis=0,
separator=" ")
self._testReduceJoin(
input_array,
truth_dim_one,
truth_shape_dim_one,
axis=1,
separator=" ")
@test_util.run_deprecated_v1
def testUnknownShape(self):
input_array = [["a"], ["b"]]
truth = ["ab"]
truth_shape = None
with self.cached_session():
placeholder = array_ops.placeholder(dtypes.string, name="placeholder")
reduced = string_ops.reduce_join(placeholder, axis=0)
output_array = reduced.eval(feed_dict={placeholder.name: input_array})
self.assertAllEqualUnicode(truth, output_array)
self.assertAllEqual(truth_shape, reduced.get_shape())
@test_util.run_deprecated_v1
def testUnknownIndices(self):
input_array = [["this", "is", "a", "test"],
["please", "do", "not", "panic"]]
truth_dim_zero = ["thisplease", "isdo", "anot", "testpanic"]
truth_dim_one = ["thisisatest", "pleasedonotpanic"]
truth_shape = None
with self.cached_session():
placeholder = array_ops.placeholder(dtypes.int32, name="placeholder")
reduced = string_ops.reduce_join(input_array, axis=placeholder)
output_array_dim_zero = reduced.eval(feed_dict={placeholder.name: [0]})
output_array_dim_one = reduced.eval(feed_dict={placeholder.name: [1]})
self.assertAllEqualUnicode(truth_dim_zero, output_array_dim_zero)
self.assertAllEqualUnicode(truth_dim_one, output_array_dim_one)
self.assertAllEqual(truth_shape, reduced.get_shape())
def testKeepDims(self):
input_array = [["this", "is", "a", "test"],
["please", "do", "not", "panic"]]
truth_dim_zero = [["thisplease", "isdo", "anot", "testpanic"]]
truth_shape_dim_zero = [1, 4]
truth_dim_one = [["thisisatest"], ["pleasedonotpanic"]]
truth_shape_dim_one = [2, 1]
self._testReduceJoin(
input_array,
truth_dim_zero,
truth_shape_dim_zero,
axis=0,
keep_dims=True)
self._testReduceJoin(
input_array,
truth_dim_one,
truth_shape_dim_one,
axis=1,
keep_dims=True)
expected_val = [["thisisatestpleasedonotpanic"]]
expected_shape = [1, 1]
self._testReduceJoin(
constant_op.constant(input_array), expected_val, expected_shape,
keep_dims=True, axis=None)
# Using axis=[] is a no-op.
expected_val = input_array
expected_shape = [2, 4]
self._testReduceJoin(
input_array, expected_val, expected_shape, keep_dims=True, axis=[])
def testMultiIndex(self):
num_dims = 3
input_array = _input_array(num_dims=num_dims)
# Also tests [].
for i in range(num_dims + 1):
for permutation in itertools.permutations(range(num_dims), i):
self._testMultipleReduceJoin(input_array, axis=permutation)
@test_util.run_deprecated_v1
def testInvalidReductionIndices(self):
with self.cached_session():
with self.assertRaisesRegex(ValueError, "Invalid reduction dim"):
string_ops.reduce_join(inputs="", axis=0)
with self.assertRaisesRegex(ValueError, "Invalid reduction dimension -3"):
string_ops.reduce_join(inputs=[[""]], axis=-3)
with self.assertRaisesRegex(ValueError, "Invalid reduction dimension 2"):
string_ops.reduce_join(inputs=[[""]], axis=2)
with self.assertRaisesRegex(ValueError, "Invalid reduction dimension -3"):
string_ops.reduce_join(inputs=[[""]], axis=[0, -3])
with self.assertRaisesRegex(ValueError, "Invalid reduction dimension 2"):
string_ops.reduce_join(inputs=[[""]], axis=[0, 2])
def testZeroDims(self):
with self.cached_session():
inputs = np.zeros([0, 1], dtype=str)
# Reduction that drops the dim of size 0.
output = string_ops.reduce_join(inputs=inputs, axis=0)
self.assertAllEqualUnicode([""], self.evaluate(output))
# Reduction that keeps the dim of size 0.
output = string_ops.reduce_join(inputs=inputs, axis=1)
output_shape = self.evaluate(output).shape
self.assertAllEqual([0], output_shape)
@test_util.run_deprecated_v1
def testInvalidArgsUnknownShape(self):
with self.cached_session():
placeholder = array_ops.placeholder(dtypes.string, name="placeholder")
index_too_high = string_ops.reduce_join(placeholder, axis=1)
duplicate_index = string_ops.reduce_join(placeholder, axis=[-1, 1])
with self.assertRaisesOpError("Invalid reduction dimension 1"):
index_too_high.eval(feed_dict={placeholder.name: [""]})
with self.assertRaisesOpError("Duplicate reduction dimension 1"):
duplicate_index.eval(feed_dict={placeholder.name: [[""]]})
@test_util.run_deprecated_v1
def testInvalidArgsUnknownIndices(self):
with self.cached_session():
placeholder = array_ops.placeholder(dtypes.int32, name="placeholder")
reduced = string_ops.reduce_join(["test", "test2"], axis=placeholder)
with self.assertRaisesOpError("reduction dimension -2"):
reduced.eval(feed_dict={placeholder.name: -2})
with self.assertRaisesOpError("reduction dimension 2"):
reduced.eval(feed_dict={placeholder.name: 2})
def testDeprecatedArgs(self):
foobar = constant_op.constant(["foobar"])
# Old names: keep_dims and reduction_indices
output = string_ops.reduce_join(
["foo", "bar"], reduction_indices=0, keep_dims=True)
self.assertAllEqual(foobar, output)
# New names keepdims and axis.
output = string_ops.reduce_join(["foo", "bar"], axis=0, keepdims=True)
self.assertAllEqual(foobar, output)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/regex_full_match_op_test.py
+95
View File
@@ -0,0 +1,95 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for RegexFullMatch op from string_ops."""
from absl.testing import parameterized
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import gen_string_ops
from tensorflow.python.ops import string_ops
from tensorflow.python.platform import test
@parameterized.parameters(
(gen_string_ops.regex_full_match),
(gen_string_ops.static_regex_full_match))
class RegexFullMatchOpVariantsTest(test.TestCase, parameterized.TestCase):
@test_util.run_deprecated_v1
def testRegexFullMatch(self, op):
values = ["abaaba", "abcdabcde"]
with self.cached_session():
input_tensor = constant_op.constant(values, dtypes.string)
matched = op(input_tensor, "a.*a").eval()
self.assertAllEqual([True, False], matched)
@test_util.run_deprecated_v1
def testRegexFullMatchTwoDims(self, op):
values = [["abaaba", "abcdabcde"], ["acdcba", "ebcda"]]
with self.cached_session():
input_tensor = constant_op.constant(values, dtypes.string)
matched = op(input_tensor, "a.*a").eval()
self.assertAllEqual([[True, False], [True, False]], matched)
@test_util.run_deprecated_v1
def testEmptyMatch(self, op):
values = ["abc", "1"]
with self.cached_session():
input_tensor = constant_op.constant(values, dtypes.string)
matched = op(input_tensor, "").eval()
self.assertAllEqual([False, False], matched)
@test_util.run_deprecated_v1
def testInvalidPattern(self, op):
values = ["abc", "1"]
with self.cached_session():
input_tensor = constant_op.constant(values, dtypes.string)
invalid_pattern = "A["
matched = op(input_tensor, invalid_pattern)
with self.assertRaisesOpError("Invalid pattern"):
self.evaluate(matched)
class RegexFullMatchOpTest(test.TestCase):
@test_util.run_deprecated_v1
def testRegexFullMatchDelegation(self):
with self.cached_session():
input_tensor = constant_op.constant("foo", dtypes.string)
pattern = "[a-z]"
op = string_ops.regex_full_match(input_tensor, pattern)
self.assertFalse(op.name.startswith("RegexFullMatch"), op.name)
pattern_tensor = constant_op.constant("[a-z]*", dtypes.string)
op_tensor = string_ops.regex_full_match(input_tensor, pattern_tensor)
self.assertTrue(op_tensor.name.startswith("RegexFullMatch"), op.name)
@test_util.run_deprecated_v1
def testStaticRegexFullMatchDelegation(self):
with self.cached_session():
input_tensor = constant_op.constant("foo", dtypes.string)
pattern = "[a-z]*"
op = string_ops.regex_full_match(input_tensor, pattern)
self.assertTrue(op.name.startswith("StaticRegexFullMatch"), op.name)
pattern_tensor = constant_op.constant("[a-z]*", dtypes.string)
op_vec = string_ops.regex_full_match(input_tensor, pattern_tensor)
self.assertTrue(op_vec.name.startswith("RegexFullMatch"), op.name)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/regex_replace_op_test.py
+123
View File
@@ -0,0 +1,123 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for RegexReplace op from string_ops."""
from absl.testing import parameterized
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import gen_string_ops
from tensorflow.python.ops import string_ops
from tensorflow.python.platform import test
@parameterized.parameters(
(gen_string_ops.regex_replace),
(gen_string_ops.static_regex_replace))
class RegexReplaceOpVariantsTest(test.TestCase, parameterized.TestCase):
@test_util.run_deprecated_v1
def testForwarding(self, op):
with self.cached_session():
# Generate an input that is uniquely consumed by the regex op.
# This exercises code paths which are optimized for this case
# (e.g., using forwarding).
inp = string_ops.substr(
constant_op.constant(["AbCdEfG",
"HiJkLmN"], dtypes.string),
pos=0,
len=5)
stripped = op(inp, "\\p{Ll}", ".")
self.assertAllEqual([b"A.C.E", b"H.J.L"], stripped)
@test_util.run_deprecated_v1
def testRemovePrefix(self, op):
values = ["a:foo", "a:bar", "a:foo", "b:baz", "b:qux", "ca:b"]
with self.cached_session():
input_vector = constant_op.constant(values, dtypes.string)
stripped = op(input_vector, "^(a:|b:)", "", replace_global=False)
self.assertAllEqual([b"foo", b"bar", b"foo", b"baz", b"qux", b"ca:b"],
stripped)
@test_util.run_deprecated_v1
def testRegexReplace(self, op):
values = ["aba\naba", "abcdabcde"]
with self.cached_session():
input_vector = constant_op.constant(values, dtypes.string)
stripped = op(input_vector, "a.*a", "(\\0)")
self.assertAllEqual([b"(aba)\n(aba)", b"(abcda)bcde"], stripped)
@test_util.run_deprecated_v1
def testEmptyMatch(self, op):
values = ["abc", "1"]
with self.cached_session():
input_vector = constant_op.constant(values, dtypes.string)
stripped = op(input_vector, "", "x")
self.assertAllEqual([b"xaxbxcx", b"x1x"], stripped)
@test_util.run_deprecated_v1
def testInvalidPattern(self, op):
values = ["abc", "1"]
with self.cached_session():
input_vector = constant_op.constant(values, dtypes.string)
invalid_pattern = "A["
replace = op(input_vector, invalid_pattern, "x")
with self.assertRaisesOpError("Invalid pattern"):
self.evaluate(replace)
@test_util.run_deprecated_v1
def testGlobal(self, op):
values = ["ababababab", "abcabcabc", ""]
with self.cached_session():
input_vector = constant_op.constant(values, dtypes.string)
stripped = op(input_vector, "ab", "abc", True)
self.assertAllEqual([b"abcabcabcabcabc", b"abccabccabcc", b""], stripped)
def as_string(s):
return s
def as_tensor(s):
return constant_op.constant(s, dtypes.string)
class RegexReplaceTest(test.TestCase, parameterized.TestCase):
@parameterized.parameters(
(as_string, as_tensor),
(as_tensor, as_string),
(as_tensor, as_tensor))
@test_util.run_deprecated_v1
def testRegexReplaceDelegation(self, pattern_fn, rewrite_fn):
with self.cached_session():
input_vector = constant_op.constant("foo", dtypes.string)
pattern = pattern_fn("[a-z]")
replace = rewrite_fn(".")
op = string_ops.regex_replace(input_vector, pattern, replace)
self.assertTrue(op.name.startswith("RegexReplace"))
@test_util.run_deprecated_v1
def testStaticRegexReplaceDelegation(self):
with self.cached_session():
input_vector = constant_op.constant("foo", dtypes.string)
pattern = "[a-z]"
replace = "."
op = string_ops.regex_replace(input_vector, pattern, replace)
self.assertTrue(op.name.startswith("StaticRegexReplace"))
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/string_bytes_split_op_test.py
+77
View File
@@ -0,0 +1,77 @@
# -*- coding: utf-8 -*-
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tf.strings.to_bytes op."""
from absl.testing import parameterized
from tensorflow.python.eager import def_function
from tensorflow.python.framework import tensor_spec
from tensorflow.python.framework import test_util
from tensorflow.python.ops.ragged import ragged_factory_ops
from tensorflow.python.ops.ragged import ragged_string_ops
from tensorflow.python.platform import test
class StringsToBytesOpTest(test_util.TensorFlowTestCase,
parameterized.TestCase):
@parameterized.parameters(
# Scalar input -> vector output
(b'hello', [b'h', b'e', b'l', b'l', b'o']),
# Vector input -> 2D ragged output
([b'hello', b'123'],
[[b'h', b'e', b'l', b'l', b'o'], [b'1', b'2', b'3']]),
# 2D tensor input -> 3D ragged output
([[b'abc', b'de'], [b'fgh', b'']],
[[[b'a', b'b', b'c'], [b'd', b'e']], [[b'f', b'g', b'h'], []]]),
# 2D ragged input -> 3D ragged output
(ragged_factory_ops.constant_value([[b'abc', b'de'], [b'f']]),
[[[b'a', b'b', b'c'], [b'd', b'e']], [[b'f']]]),
# 3D input -> 4D ragged output
(ragged_factory_ops.constant_value(
[[[b'big', b'small'], [b'red']], [[b'cat', b'dog'], [b'ox']]]),
[[[[b'b', b'i', b'g'], [b's', b'm', b'a', b'l', b'l']],
[[b'r', b'e', b'd']]],
[[[b'c', b'a', b't'], [b'd', b'o', b'g']],
[[b'o', b'x']]]]),
# Empty string
(b'', []),
# Null byte
(b'\x00', [b'\x00']),
# Unicode
(u'仅今年前'.encode('utf-8'),
[b'\xe4', b'\xbb', b'\x85', b'\xe4', b'\xbb', b'\x8a', b'\xe5',
b'\xb9', b'\xb4', b'\xe5', b'\x89', b'\x8d']),
)
def testStringToBytes(self, source, expected):
expected = ragged_factory_ops.constant_value(expected, dtype=object)
result = ragged_string_ops.string_bytes_split(source)
self.assertAllEqual(expected, result)
def testUnknownInputRankError(self):
# Use a tf.function that erases shape information.
@def_function.function(input_signature=[tensor_spec.TensorSpec(None)])
def f(v):
return ragged_string_ops.string_bytes_split(v)
with self.assertRaisesRegex(TypeError,
'Binding inputs to tf.function failed'):
f(['foo'])
if __name__ == '__main__':
test.main()
tensorflow/python/kernel_tests/strings_ops/string_format_op_test.py
+389
View File
@@ -0,0 +1,389 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tensorflow.kernels.logging_ops."""
from tensorflow.python.eager import context
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import string_ops
from tensorflow.python.ops import variables
from tensorflow.python.platform import test
from tensorflow.python.util import compat
class StringFormatOpTest(test.TestCase):
@test_util.run_in_graph_and_eager_modes()
def testFormatOneTensorOneDim(self):
with self.cached_session():
tensor = math_ops.range(10)
format_output = string_ops.string_format("{}", tensor)
out = self.evaluate(format_output)
expected = "[0 1 2 ... 7 8 9]"
self.assertEqual(compat.as_text(out), expected)
with self.cached_session():
tensor = math_ops.range(10)
format_output = string_ops.string_format("{}", [tensor])
out = self.evaluate(format_output)
expected = "[0 1 2 ... 7 8 9]"
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatOneVariableScalar(self):
with self.cached_session():
var = variables.Variable(3.34)
format_output = string_ops.string_format("{}", [var])
if not context.executing_eagerly():
self.evaluate(variables.global_variables_initializer())
out = self.evaluate(format_output)
expected = "3.34"
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatOneVariableOneDim(self):
with self.cached_session():
var = variables.Variable(math_ops.range(10))
format_output = string_ops.string_format("{}", [var])
if not context.executing_eagerly():
self.evaluate(variables.global_variables_initializer())
out = self.evaluate(format_output)
expected = "[0 1 2 ... 7 8 9]"
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatTwoVariablesWithAssignAdd(self):
with self.cached_session():
var_one = variables.Variable(2.14)
plus_one = var_one.assign_add(1.0)
var_two = variables.Variable(math_ops.range(10))
format_output = string_ops.string_format("{}, {}", [var_one, var_two])
if not context.executing_eagerly():
self.evaluate(variables.global_variables_initializer())
self.evaluate(plus_one)
out = self.evaluate(format_output)
expected = "3.14, [0 1 2 ... 7 8 9]"
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatOneTensorOneDimFloat(self):
with self.cached_session():
tensor = constant_op.constant([0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7])
format_output = string_ops.string_format("{}", tensor)
out = self.evaluate(format_output)
expected = "[0 0.1 0.2 ... 0.5 0.6 0.7]"
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatOneTensorOneDimMatchesSummarize(self):
with self.cached_session():
tensor = math_ops.range(6)
format_output = string_ops.string_format("{}", tensor, summarize=3)
out = self.evaluate(format_output)
expected = "[0 1 2 3 4 5]"
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatOneTensorOneDimVarySummarize(self):
with self.cached_session():
tensor = math_ops.range(6)
format_output = string_ops.string_format("{}", tensor, summarize=-1)
out = self.evaluate(format_output)
expected = "[0 1 2 3 4 5]"
self.assertEqual(compat.as_text(out), expected)
with self.cached_session():
tensor = math_ops.range(6)
format_output = string_ops.string_format("{}", tensor, summarize=1)
out = self.evaluate(format_output)
expected = "[0 ... 5]"
self.assertEqual(compat.as_text(out), expected)
with self.cached_session():
tensor = math_ops.range(6)
format_output = string_ops.string_format("{}", tensor, summarize=2)
out = self.evaluate(format_output)
expected = "[0 1 ... 4 5]"
self.assertEqual(compat.as_text(out), expected)
with self.cached_session():
tensor = math_ops.range(6)
format_output = string_ops.string_format("{}", tensor, summarize=10)
out = self.evaluate(format_output)
expected = "[0 1 2 3 4 5]"
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatOneTensorOneDimAlmostSummarize(self):
with self.cached_session():
tensor = math_ops.range(5)
format_output = string_ops.string_format("{}", tensor, summarize=3)
out = self.evaluate(format_output)
expected = "[0 1 2 3 4]"
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatOneTensorTwoDimLessThanSummarize(self):
with self.cached_session():
tensor = array_ops.reshape(math_ops.range(4), [2, 2])
format_output = string_ops.string_format("{}", tensor, summarize=3)
out = self.evaluate(format_output)
expected = ("[[0 1]\n"
" [2 3]]")
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatOneTensorTwoDim(self):
with self.cached_session():
tensor = array_ops.reshape(math_ops.range(100), [10, 10])
format_output = string_ops.string_format("{}", tensor)
out = self.evaluate(format_output)
expected = ("[[0 1 2 ... 7 8 9]\n"
" [10 11 12 ... 17 18 19]\n"
" [20 21 22 ... 27 28 29]\n"
" ...\n"
" [70 71 72 ... 77 78 79]\n"
" [80 81 82 ... 87 88 89]\n"
" [90 91 92 ... 97 98 99]]")
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatOneTensorTwoDimSummarizeTwo(self):
with self.cached_session():
tensor = array_ops.reshape(math_ops.range(100), [10, 10])
format_output = string_ops.string_format("{}", tensor, summarize=2)
out = self.evaluate(format_output)
expected = ("[[0 1 ... 8 9]\n"
" [10 11 ... 18 19]\n"
" ...\n"
" [80 81 ... 88 89]\n"
" [90 91 ... 98 99]]")
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatOneTensorThreeDim(self):
with self.cached_session():
tensor = array_ops.reshape(math_ops.range(1000), [10, 10, 10])
format_output = string_ops.string_format("{}", tensor)
out = self.evaluate(format_output)
expected = ("[[[0 1 2 ... 7 8 9]\n"
" [10 11 12 ... 17 18 19]\n"
" [20 21 22 ... 27 28 29]\n"
" ...\n"
" [70 71 72 ... 77 78 79]\n"
" [80 81 82 ... 87 88 89]\n"
" [90 91 92 ... 97 98 99]]\n"
"\n"
" [[100 101 102 ... 107 108 109]\n"
" [110 111 112 ... 117 118 119]\n"
" [120 121 122 ... 127 128 129]\n"
" ...\n [170 171 172 ... 177 178 179]\n"
" [180 181 182 ... 187 188 189]\n"
" [190 191 192 ... 197 198 199]]\n"
"\n"
" [[200 201 202 ... 207 208 209]\n"
" [210 211 212 ... 217 218 219]\n"
" [220 221 222 ... 227 228 229]\n"
" ...\n"
" [270 271 272 ... 277 278 279]\n"
" [280 281 282 ... 287 288 289]\n"
" [290 291 292 ... 297 298 299]]\n"
"\n"
" ...\n"
"\n"
" [[700 701 702 ... 707 708 709]\n"
" [710 711 712 ... 717 718 719]\n"
" [720 721 722 ... 727 728 729]\n"
" ...\n"
" [770 771 772 ... 777 778 779]\n"
" [780 781 782 ... 787 788 789]\n"
" [790 791 792 ... 797 798 799]]\n"
"\n"
" [[800 801 802 ... 807 808 809]\n"
" [810 811 812 ... 817 818 819]\n"
" [820 821 822 ... 827 828 829]\n"
" ...\n"
" [870 871 872 ... 877 878 879]\n"
" [880 881 882 ... 887 888 889]\n"
" [890 891 892 ... 897 898 899]]\n"
"\n"
" [[900 901 902 ... 907 908 909]\n"
" [910 911 912 ... 917 918 919]\n"
" [920 921 922 ... 927 928 929]\n"
" ...\n"
" [970 971 972 ... 977 978 979]\n"
" [980 981 982 ... 987 988 989]\n"
" [990 991 992 ... 997 998 999]]]")
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatOneTensorTemplatePrefix(self):
with self.cached_session():
tensor = array_ops.reshape(math_ops.range(100), [10, 10])
format_output = string_ops.string_format("tensor summary: {}", tensor)
out = self.evaluate(format_output)
expected = ("tensor summary: [[0 1 2 ... 7 8 9]\n"
" [10 11 12 ... 17 18 19]\n"
" [20 21 22 ... 27 28 29]\n"
" ...\n"
" [70 71 72 ... 77 78 79]\n"
" [80 81 82 ... 87 88 89]\n"
" [90 91 92 ... 97 98 99]]")
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatOneTensorTemplatePrefixAndSuffix(self):
with self.cached_session():
tensor = array_ops.reshape(math_ops.range(100), [10, 10])
format_output = string_ops.string_format("tensor summary: {}, suffix",
tensor)
out = self.evaluate(format_output)
expected = ("tensor summary: [[0 1 2 ... 7 8 9]\n"
" [10 11 12 ... 17 18 19]\n"
" [20 21 22 ... 27 28 29]\n"
" ...\n"
" [70 71 72 ... 77 78 79]\n"
" [80 81 82 ... 87 88 89]\n"
" [90 91 92 ... 97 98 99]], suffix")
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatOneTensorTemplateSuffix(self):
with self.cached_session():
tensor = array_ops.reshape(math_ops.range(100), [10, 10])
format_output = string_ops.string_format("{}, suffix", tensor)
out = self.evaluate(format_output)
expected = ("[[0 1 2 ... 7 8 9]\n"
" [10 11 12 ... 17 18 19]\n"
" [20 21 22 ... 27 28 29]\n"
" ...\n"
" [70 71 72 ... 77 78 79]\n"
" [80 81 82 ... 87 88 89]\n"
" [90 91 92 ... 97 98 99]], suffix")
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatNoTensor(self):
with self.cached_session():
format_output = string_ops.string_format("No tensor.", ())
out = self.evaluate(format_output)
expected = "No tensor."
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatMultiTensor(self):
with self.cached_session():
tensor_one = array_ops.reshape(math_ops.range(100), [10, 10])
tensor_two = tensor_one * 10
format_output = string_ops.string_format("One: {},\nTwo: {}",
(tensor_one, tensor_two))
out = self.evaluate(format_output)
expected = ("One: [[0 1 2 ... 7 8 9]\n"
" [10 11 12 ... 17 18 19]\n"
" [20 21 22 ... 27 28 29]\n"
" ...\n"
" [70 71 72 ... 77 78 79]\n"
" [80 81 82 ... 87 88 89]\n"
" [90 91 92 ... 97 98 99]],\n"
"Two: [[0 10 20 ... 70 80 90]\n"
" [100 110 120 ... 170 180 190]\n"
" [200 210 220 ... 270 280 290]\n"
" ...\n"
" [700 710 720 ... 770 780 790]\n"
" [800 810 820 ... 870 880 890]\n"
" [900 910 920 ... 970 980 990]]")
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatSummarizeOne(self):
with self.cached_session():
tensor = array_ops.reshape(math_ops.range(100), [10, 10])
format_output = string_ops.string_format("tensor summary: {}", tensor,
summarize=1)
out = self.evaluate(format_output)
expected = ("tensor summary: [[0 ... 9]\n"
" ...\n"
" [90 ... 99]]")
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatSummarizeTwo(self):
with self.cached_session():
tensor = array_ops.reshape(math_ops.range(100), [10, 10])
format_output = string_ops.string_format("tensor summary: {}", tensor,
summarize=2)
out = self.evaluate(format_output)
expected = ("tensor summary: [[0 1 ... 8 9]\n"
" [10 11 ... 18 19]\n"
" ...\n"
" [80 81 ... 88 89]\n"
" [90 91 ... 98 99]]")
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testFormatPlaceholder(self):
with self.cached_session():
tensor = array_ops.reshape(math_ops.range(100), [10, 10])
format_output = string_ops.string_format("tensor summary: %t%", tensor,
placeholder="%t%")
out = self.evaluate(format_output)
expected = ("tensor summary: [[0 1 2 ... 7 8 9]\n"
" [10 11 12 ... 17 18 19]\n"
" [20 21 22 ... 27 28 29]\n"
" ...\n"
" [70 71 72 ... 77 78 79]\n"
" [80 81 82 ... 87 88 89]\n"
" [90 91 92 ... 97 98 99]]")
self.assertEqual(compat.as_text(out), expected)
@test_util.run_in_graph_and_eager_modes()
def testTensorCountMustMatchPlaceholderCount(self):
with self.cached_session():
with self.assertRaisesRegex(
ValueError, r"The template expects 2 tensors, but the inputs only has"
r" 1\.\s.*"):
tensor = math_ops.range(10)
format_output = string_ops.string_format("{} {}", tensor)
self.evaluate(format_output)
with self.cached_session():
with self.assertRaisesRegex(
ValueError, r"The template expects 2 tensors, but the inputs only has"
r" 1\.\s.*"):
tensor = math_ops.range(10)
format_output = string_ops.string_format("{} {}", [tensor])
self.evaluate(format_output)
with self.cached_session():
with self.assertRaisesRegex(
ValueError, r"The template expects 1 tensors, but the inputs only has"
r" 2\.\s.*"):
tensor = math_ops.range(10)
format_output = string_ops.string_format("{}", (tensor, tensor))
self.evaluate(format_output)
@test_util.run_in_graph_and_eager_modes()
def testTensorAndFormatUnicode(self):
with self.cached_session():
tensor = constant_op.constant("😊")
format_output = string_ops.string_format("😊:{}", tensor)
out = self.evaluate(format_output)
expected = '😊:"😊"'
self.assertEqual(compat.as_text(out), expected)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/string_join_op_test.py
+54
View File
@@ -0,0 +1,54 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for string_join_op."""
from tensorflow.python.framework import errors
from tensorflow.python.ops import string_ops
from tensorflow.python.platform import test
class StringJoinOpTest(test.TestCase):
def testStringJoin(self):
input0 = ["a", "b"]
input1 = "a"
input2 = [["b"], ["c"]]
output = string_ops.string_join([input0, input1])
self.assertAllEqual(output, [b"aa", b"ba"])
output = string_ops.string_join([input0, input1], separator="--")
self.assertAllEqual(output, [b"a--a", b"b--a"])
output = string_ops.string_join([input0, input1, input0], separator="--")
self.assertAllEqual(output, [b"a--a--a", b"b--a--b"])
output = string_ops.string_join([input1] * 4, separator="!")
self.assertEqual(self.evaluate(output), b"a!a!a!a")
output = string_ops.string_join([input2] * 2, separator="")
self.assertAllEqual(output, [[b"bb"], [b"cc"]])
output = string_ops.string_join([])
self.assertAllEqual(output, b"")
with self.assertRaisesRegex(
(ValueError, errors.InvalidArgumentError),
"shapes do not match|must be equal rank",
):
self.evaluate(string_ops.string_join([input0, input2]))
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/string_length_op_test.py
+63
View File
@@ -0,0 +1,63 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for string_length_op."""
from tensorflow.python.framework import test_util
from tensorflow.python.ops import string_ops
from tensorflow.python.platform import test
class StringLengthOpTest(test.TestCase):
def testStringLength(self):
strings = [[["1", "12"], ["123", "1234"], ["12345", "123456"]]]
with self.cached_session() as sess:
lengths = string_ops.string_length(strings)
values = self.evaluate(lengths)
self.assertAllEqual(values, [[[1, 2], [3, 4], [5, 6]]])
@test_util.run_deprecated_v1
def testUnit(self):
unicode_strings = [u"H\xc3llo", u"\U0001f604"]
utf8_strings = [s.encode("utf-8") for s in unicode_strings]
expected_utf8_byte_lengths = [6, 4]
expected_utf8_char_lengths = [5, 1]
with self.session() as sess:
utf8_byte_lengths = string_ops.string_length(utf8_strings, unit="BYTE")
utf8_char_lengths = string_ops.string_length(
utf8_strings, unit="UTF8_CHAR")
self.assertAllEqual(
self.evaluate(utf8_byte_lengths), expected_utf8_byte_lengths)
self.assertAllEqual(
self.evaluate(utf8_char_lengths), expected_utf8_char_lengths)
with self.assertRaisesRegex(
ValueError, "Attr 'unit' of 'StringLength' Op passed string 'XYZ' "
'not in: "BYTE", "UTF8_CHAR"'):
string_ops.string_length(utf8_strings, unit="XYZ")
@test_util.run_deprecated_v1
def testLegacyPositionalName(self):
# Code that predates the 'unit' parameter may have used a positional
# argument for the 'name' parameter. Check that we don't break such code.
strings = [[["1", "12"], ["123", "1234"], ["12345", "123456"]]]
lengths = string_ops.string_length(strings, "some_name")
with self.session():
self.assertAllEqual(lengths, [[[1, 2], [3, 4], [5, 6]]])
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/string_lower_op_test.py
+52
View File
@@ -0,0 +1,52 @@
# -*- coding: utf-8 -*-
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for string_lower_op."""
from tensorflow.python.ops import string_ops
from tensorflow.python.platform import test
class StringLowerOpTest(test.TestCase):
"""Test cases for tf.strings.lower."""
def test_string_lower(self):
strings = ["Pigs on The Wing", "aNimals"]
with self.cached_session():
output = string_ops.string_lower(strings)
output = self.evaluate(output)
self.assertAllEqual(output, [b"pigs on the wing", b"animals"])
def test_string_lower_2d(self):
strings = [["pigS on THE wIng", "aniMals"], [" hello ", "\n\tWorld! \r \n"]]
with self.cached_session():
output = string_ops.string_lower(strings)
output = self.evaluate(output)
self.assertAllEqual(output, [[b"pigs on the wing", b"animals"],
[b" hello ", b"\n\tworld! \r \n"]])
def test_string_upper_unicode(self):
strings = [["ÓÓSSCHLOË"]]
with self.cached_session():
output = string_ops.string_lower(strings, encoding="utf-8")
output = self.evaluate(output)
# output: "óósschloë"
self.assertAllEqual(output, [[b"\xc3\xb3\xc3\xb3sschlo\xc3\xab"]])
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/string_split_op_test.py
+461
View File
@@ -0,0 +1,461 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for string_split_op."""
from absl.testing import parameterized
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import sparse_tensor
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import string_ops
from tensorflow.python.ops.ragged import ragged_factory_ops
from tensorflow.python.ops.ragged import ragged_string_ops
from tensorflow.python.platform import test
from tensorflow.python.util import compat
class StringSplitOpTest(test.TestCase, parameterized.TestCase):
def testStringSplit(self):
strings = ["pigs on the wing", "animals"]
with self.cached_session():
tokens = string_ops.string_split(strings)
indices, values, shape = self.evaluate(tokens)
self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [1, 0]])
self.assertAllEqual(values, [b"pigs", b"on", b"the", b"wing", b"animals"])
self.assertAllEqual(shape, [2, 4])
@test_util.run_deprecated_v1
def testStringSplitEmptyDelimiter(self):
strings = ["hello", "hola", b"\xF0\x9F\x98\x8E"] # Last string is U+1F60E
with self.cached_session():
tokens = string_ops.string_split(strings, delimiter="")
indices, values, shape = self.evaluate(tokens)
self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [0, 4],
[1, 0], [1, 1], [1, 2], [1, 3], [2, 0],
[2, 1], [2, 2], [2, 3]])
expected = np.array(
[
"h", "e", "l", "l", "o", "h", "o", "l", "a", b"\xf0", b"\x9f",
b"\x98", b"\x8e"
],
dtype="|S1")
self.assertAllEqual(values.tolist(), expected)
self.assertAllEqual(shape, [3, 5])
def testStringSplitEmptyToken(self):
strings = ["", " a", "b ", " c", " ", " d ", " e", "f ", " g ", " "]
with self.cached_session():
tokens = string_ops.string_split(strings)
indices, values, shape = self.evaluate(tokens)
self.assertAllEqual(
indices,
[[1, 0], [2, 0], [3, 0], [5, 0], [6, 0], [7, 0], [8, 0]])
self.assertAllEqual(values, [b"a", b"b", b"c", b"d", b"e", b"f", b"g"])
self.assertAllEqual(shape, [10, 1])
def testStringSplitOnSetEmptyToken(self):
strings = ["", " a", "b ", " c", " ", " d ", ". e", "f .", " .g. ", " ."]
with self.cached_session():
tokens = string_ops.string_split(strings, delimiter=" .")
indices, values, shape = self.evaluate(tokens)
self.assertAllEqual(
indices,
[[1, 0], [2, 0], [3, 0], [5, 0], [6, 0], [7, 0], [8, 0]])
self.assertAllEqual(values, [b"a", b"b", b"c", b"d", b"e", b"f", b"g"])
self.assertAllEqual(shape, [10, 1])
@test_util.run_deprecated_v1
def testStringSplitWithDelimiter(self):
strings = ["hello|world", "hello world"]
with self.cached_session():
self.assertRaises(
ValueError, string_ops.string_split, strings, delimiter=["|", ""])
self.assertRaises(
ValueError, string_ops.string_split, strings, delimiter=["a"])
tokens = string_ops.string_split(strings, delimiter="|")
indices, values, shape = self.evaluate(tokens)
self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0]])
self.assertAllEqual(values, [b"hello", b"world", b"hello world"])
self.assertAllEqual(shape, [2, 2])
tokens = string_ops.string_split(strings, delimiter="| ")
indices, values, shape = self.evaluate(tokens)
self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0], [1, 1]])
self.assertAllEqual(values, [b"hello", b"world", b"hello", b"world"])
self.assertAllEqual(shape, [2, 2])
@test_util.run_deprecated_v1
def testStringSplitWithDelimiterTensor(self):
strings = ["hello|world", "hello world"]
with self.cached_session() as sess:
delimiter = array_ops.placeholder(dtypes.string)
tokens = string_ops.string_split(strings, delimiter=delimiter)
with self.assertRaises(errors_impl.InvalidArgumentError):
sess.run(tokens, feed_dict={delimiter: ["a", "b"]})
with self.assertRaises(errors_impl.InvalidArgumentError):
sess.run(tokens, feed_dict={delimiter: ["a"]})
indices, values, shape = sess.run(tokens, feed_dict={delimiter: "|"})
self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0]])
self.assertAllEqual(values, [b"hello", b"world", b"hello world"])
self.assertAllEqual(shape, [2, 2])
@test_util.run_deprecated_v1
def testStringSplitWithDelimitersTensor(self):
strings = ["hello.cruel,world", "hello cruel world"]
with self.cached_session() as sess:
delimiter = array_ops.placeholder(dtypes.string)
tokens = string_ops.string_split(strings, delimiter=delimiter)
with self.assertRaises(errors_impl.InvalidArgumentError):
sess.run(tokens, feed_dict={delimiter: ["a", "b"]})
with self.assertRaises(errors_impl.InvalidArgumentError):
sess.run(tokens, feed_dict={delimiter: ["a"]})
indices, values, shape = sess.run(tokens, feed_dict={delimiter: ".,"})
self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [1, 0]])
self.assertAllEqual(values,
[b"hello", b"cruel", b"world", b"hello cruel world"])
self.assertAllEqual(shape, [2, 3])
def testStringSplitWithNoSkipEmpty(self):
strings = ["#a", "b#", "#c#"]
with self.cached_session():
tokens = string_ops.string_split(strings, "#", skip_empty=False)
indices, values, shape = self.evaluate(tokens)
self.assertAllEqual(indices, [[0, 0], [0, 1],
[1, 0], [1, 1],
[2, 0], [2, 1], [2, 2]])
self.assertAllEqual(values, [b"", b"a", b"b", b"", b"", b"c", b""])
self.assertAllEqual(shape, [3, 3])
with self.cached_session():
tokens = string_ops.string_split(strings, "#")
indices, values, shape = self.evaluate(tokens)
self.assertAllEqual(values, [b"a", b"b", b"c"])
self.assertAllEqual(indices, [[0, 0], [1, 0], [2, 0]])
self.assertAllEqual(shape, [3, 1])
@parameterized.named_parameters([
dict(
testcase_name="RaggedResultType",
source=[b"pigs on the wing", b"animals"],
result_type="RaggedTensor",
expected=[[b"pigs", b"on", b"the", b"wing"], [b"animals"]]),
dict(
testcase_name="SparseResultType",
source=[b"pigs on the wing", b"animals"],
result_type="SparseTensor",
expected=sparse_tensor.SparseTensorValue(
[[0, 0], [0, 1], [0, 2], [0, 3], [1, 0]],
[b"pigs", b"on", b"the", b"wing", b"animals"], [2, 4])),
dict(
testcase_name="DefaultResultType",
source=[b"pigs on the wing", b"animals"],
expected=sparse_tensor.SparseTensorValue(
[[0, 0], [0, 1], [0, 2], [0, 3], [1, 0]],
[b"pigs", b"on", b"the", b"wing", b"animals"], [2, 4])),
dict(
testcase_name="BadResultType",
source=[b"pigs on the wing", b"animals"],
result_type="BouncyTensor",
error="result_type must be .*"),
dict(
testcase_name="WithSepAndAndSkipEmpty",
source=[b"+hello+++this+is+a+test"],
sep="+",
skip_empty=False,
result_type="RaggedTensor",
expected=[[b"", b"hello", b"", b"", b"this", b"is", b"a", b"test"]]),
dict(
testcase_name="WithDelimiter",
source=[b"hello world"],
delimiter="l",
result_type="RaggedTensor",
expected=[[b"he", b"o wor", b"d"]]),
])
def testRaggedStringSplitWrapper(self,
source,
sep=None,
skip_empty=True,
delimiter=None,
result_type="SparseTensor",
expected=None,
error=None):
if error is not None:
with self.assertRaisesRegex(ValueError, error):
ragged_string_ops.string_split(source, sep, skip_empty, delimiter,
result_type)
if expected is not None:
result = ragged_string_ops.string_split(source, sep, skip_empty,
delimiter, result_type)
if isinstance(expected, sparse_tensor.SparseTensorValue):
self.assertAllEqual(result.indices, expected.indices)
self.assertAllEqual(result.values, expected.values)
self.assertAllEqual(result.dense_shape, expected.dense_shape)
else:
self.assertAllEqual(result, expected)
class StringSplitV2OpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
@parameterized.named_parameters([
{"testcase_name": "Simple",
"input": [b"pigs on the wing", b"animals"],
"expected": [[b"pigs", b"on", b"the", b"wing"], [b"animals"]]},
{"testcase_name": "MultiCharSeparator",
"input": [b"1<>2<>3", b"<><>4<>5<><>6<>"],
"sep": b"<>",
"expected": [[b"1", b"2", b"3"],
[b"", b"", b"4", b"5", b"", b"6", b""]]},
{"testcase_name": "SimpleSeparator",
"input": [b"1,2,3", b"4,5,,6,"],
"sep": b",",
"expected": [[b"1", b"2", b"3"], [b"4", b"5", b"", b"6", b""]]},
{"testcase_name": "EmptySeparator",
"input": [b"1 2 3", b" 4 5 6 "],
"expected": [[b"1", b"2", b"3"], [b"4", b"5", b"6"]]},
{"testcase_name": "EmptySeparatorEmptyInputString",
"input": [b""],
"expected": [[]]},
{"testcase_name": "EmptyInputVector",
"input": [],
"expected": []},
{"testcase_name": "SimpleSeparatorMaxSplit",
"input": [b"1,2,3", b"4,5,,6,"],
"sep": b",",
"maxsplit": 1,
"expected": [[b"1", b"2,3"], [b"4", b"5,,6,"]]},
{"testcase_name": "EmptySeparatorMaxSplit",
"input": [b"1 2 3", b" 4 5 6 "],
"maxsplit": 1,
"expected": [[b"1", b"2 3"], [b"4", b"5 6 "]]},
{"testcase_name": "ScalarInput",
"input": b"1,2,3",
"sep": b",",
"expected": [b"1", b"2", b"3"]},
{"testcase_name": "Dense2DInput",
"input": [[b"1,2,3", b"4"], [b"5,6", b"7,8,9"]],
"sep": b",",
"expected": [[[b"1", b"2", b"3"], [b"4"]],
[[b"5", b"6"], [b"7", b"8", b"9"]]]},
{"testcase_name": "Ragged2DInput",
"input": [[b"1,2,3", b"4"], [b"5,6"]],
"input_is_ragged": True,
"sep": b",",
"expected": [[[b"1", b"2", b"3"], [b"4"]], [[b"5", b"6"]]]},
{"testcase_name": "Ragged3DInput",
"input": [[[b"1,2,3", b"4"], [b"5,6"]], [[b"7,8,9"]]],
"input_is_ragged": True,
"sep": b",",
"expected": [[[[b"1", b"2", b"3"], [b"4"]], [[b"5", b"6"]]],
[[[b"7", b"8", b"9"]]]]},
{"testcase_name": "Ragged4DInput",
"input": [[[[b"1,2,3", b"4"], [b"5,6"]], [[b"7,8,9"]]], [[[b""]]]],
"input_is_ragged": True,
"sep": b",",
"expected": [[[[[b"1", b"2", b"3"], [b"4"]], [[b"5", b"6"]]],
[[[b"7", b"8", b"9"]]]], [[[[b""]]]]]},
{"testcase_name": "Ragged4DInputEmptySeparator",
"input": [[[[b"1 2 3", b"4"], [b"5 6"]], [[b"7 8 9"]]], [[[b""]]]],
"input_is_ragged": True,
"expected": [[[[[b"1", b"2", b"3"], [b"4"]], [[b"5", b"6"]]],
[[[b"7", b"8", b"9"]]]], [[[[]]]]]},
]) # pyformat: disable
def testSplitV2(self,
input,
expected,
input_is_ragged=False,
**kwargs): # pylint: disable=redefined-builtin
# Check that we are matching the behavior of Python's str.split:
self.assertEqual(expected, self._py_split(input, **kwargs))
# Prepare the input tensor.
if input_is_ragged:
input = ragged_factory_ops.constant(input, dtype=dtypes.string)
else:
input = constant_op.constant(input, dtype=dtypes.string)
# Check that the public version (which returns a RaggedTensor) works
# correctly.
expected_ragged = ragged_factory_ops.constant(
expected, ragged_rank=input.shape.ndims)
actual_ragged_v2 = ragged_string_ops.string_split_v2(input, **kwargs)
actual_ragged_v2_input_kwarg = ragged_string_ops.string_split_v2(
input=input, **kwargs)
self.assertAllEqual(expected_ragged, actual_ragged_v2)
self.assertAllEqual(expected_ragged, actual_ragged_v2_input_kwarg)
# Check that the internal version (which returns a SparseTensor) works
# correctly. Note: the internal version oly supports vector inputs.
if input.shape.ndims == 1:
expected_sparse = self.evaluate(expected_ragged.to_sparse())
actual_sparse_v2 = string_ops.string_split_v2(input, **kwargs)
self.assertEqual(expected_sparse.indices.tolist(),
self.evaluate(actual_sparse_v2.indices).tolist())
self.assertEqual(expected_sparse.values.tolist(),
self.evaluate(actual_sparse_v2.values).tolist())
self.assertEqual(expected_sparse.dense_shape.tolist(),
self.evaluate(actual_sparse_v2.dense_shape).tolist())
@parameterized.named_parameters([
{"testcase_name": "Simple",
"input": [b"pigs on the wing", b"animals"],
"expected": [[b"pigs", b"on", b"the", b"wing"], [b"animals"]]},
{"testcase_name": "MultiCharSeparator",
"input": [b"1<>2<>3", b"<><>4<>5<><>6<>"],
"sep": b"<>",
"expected": [[b"1", b"2", b"3"],
[b"", b"", b"4", b"5", b"", b"6", b""]]},
{"testcase_name": "SimpleSeparator",
"input": [b"1,2,3", b"4,5,,6,"],
"sep": b",",
"expected": [[b"1", b"2", b"3"], [b"4", b"5", b"", b"6", b""]]},
{"testcase_name": "EmptySeparator",
"input": [b"1 2 3", b" 4 5 6 "],
"expected": [[b"1", b"2", b"3"], [b"4", b"5", b"6"]]},
{"testcase_name": "EmptySeparatorEmptyInputString",
"input": [b""],
"expected": [[]]},
{"testcase_name": "SimpleSeparatorMaxSplit",
"input": [b"1,2,3", b"4,5,,6,"],
"sep": b",",
"maxsplit": 1,
"expected": [[b"1", b"2,3"], [b"4", b"5,,6,"]]},
{"testcase_name": "EmptySeparatorMaxSplit",
"input": [b"1 2 3", b" 4 5 6 "],
"maxsplit": 1,
"expected": [[b"1", b"2 3"], [b"4", b"5 6 "]]},
{"testcase_name": "ScalarInput",
"input": b"1,2,3",
"sep": b",",
"expected": [[b"1", b"2", b"3"]]},
{"testcase_name": "Dense2DInput",
"input": [[b"1,2,3", b"4"], [b"5,6", b"7,8,9"]],
"sep": b",",
"expected": [[[b"1", b"2", b"3"], [b"4"]],
[[b"5", b"6"], [b"7", b"8", b"9"]]]},
{"testcase_name": "Ragged2DInput",
"input": [[b"1,2,3", b"4"], [b"5,6"]],
"input_is_ragged": True,
"sep": b",",
"expected": [[[b"1", b"2", b"3"], [b"4"]], [[b"5", b"6"]]]},
{"testcase_name": "Ragged3DInput",
"input": [[[b"1,2,3", b"4"], [b"5,6"]], [[b"7,8,9"]]],
"input_is_ragged": True,
"sep": b",",
"expected": [[[[b"1", b"2", b"3"], [b"4"]], [[b"5", b"6"]]],
[[[b"7", b"8", b"9"]]]]},
{"testcase_name": "Ragged4DInput",
"input": [[[[b"1,2,3", b"4"], [b"5,6"]], [[b"7,8,9"]]], [[[b""]]]],
"input_is_ragged": True,
"sep": b",",
"expected": [[[[[b"1", b"2", b"3"], [b"4"]], [[b"5", b"6"]]],
[[[b"7", b"8", b"9"]]]], [[[[b""]]]]]},
{"testcase_name": "Ragged4DInputEmptySeparator",
"input": [[[[b"1 2 3", b"4"], [b"5 6"]], [[b"7 8 9"]]], [[[b""]]]],
"input_is_ragged": True,
"expected": [[[[[b"1", b"2", b"3"], [b"4"]], [[b"5", b"6"]]],
[[[b"7", b"8", b"9"]]]], [[[[]]]]]},
]) # pyformat: disable
def testSplitV1(self, input, expected, input_is_ragged=False, **kwargs): # pylint: disable=redefined-builtin
# Prepare the input tensor.
if input_is_ragged:
input = ragged_factory_ops.constant(input, dtype=dtypes.string)
else:
input = constant_op.constant(input, dtype=dtypes.string)
expected_ragged = ragged_factory_ops.constant(expected)
actual_ragged_v1 = ragged_string_ops.strings_split_v1(
input, result_type="RaggedTensor", **kwargs)
actual_ragged_v1_input_kwarg = ragged_string_ops.strings_split_v1(
input=input, result_type="RaggedTensor", **kwargs)
actual_ragged_v1_source_kwarg = ragged_string_ops.strings_split_v1(
source=input, result_type="RaggedTensor", **kwargs)
self.assertAllEqual(expected_ragged, actual_ragged_v1)
self.assertAllEqual(expected_ragged, actual_ragged_v1_input_kwarg)
self.assertAllEqual(expected_ragged, actual_ragged_v1_source_kwarg)
expected_sparse = self.evaluate(expected_ragged.to_sparse())
actual_sparse_v1 = ragged_string_ops.strings_split_v1(
input, result_type="SparseTensor", **kwargs)
self.assertEqual(expected_sparse.indices.tolist(),
self.evaluate(actual_sparse_v1.indices).tolist())
self.assertEqual(expected_sparse.values.tolist(),
self.evaluate(actual_sparse_v1.values).tolist())
self.assertEqual(expected_sparse.dense_shape.tolist(),
self.evaluate(actual_sparse_v1.dense_shape).tolist())
def testSplitV1BadResultType(self):
with self.assertRaisesRegex(ValueError, "result_type must be .*"):
ragged_string_ops.strings_split_v1("foo", result_type="BouncyTensor")
def _py_split(self, strings, **kwargs):
if isinstance(strings, compat.bytes_or_text_types):
# Note: str.split doesn't accept keyword args.
if "maxsplit" in kwargs:
return strings.split(kwargs.get("sep", None), kwargs["maxsplit"])
else:
return strings.split(kwargs.get("sep", None))
else:
return [self._py_split(s, **kwargs) for s in strings]
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/string_strip_op_test.py
+52
View File
@@ -0,0 +1,52 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for string_strip_op."""
from tensorflow.python.ops import string_ops
from tensorflow.python.platform import test
class StringStripOpTest(test.TestCase):
""" Test cases for tf.strings.strip."""
def test_string_strip(self):
strings = ["pigs on the wing", "animals"]
with self.cached_session() as sess:
output = string_ops.string_strip(strings)
output = self.evaluate(output)
self.assertAllEqual(output, [b"pigs on the wing", b"animals"])
def test_string_strip_2d(self):
strings = [["pigs on the wing", "animals"],
[" hello ", "\n\tworld \r \n"]]
with self.cached_session() as sess:
output = string_ops.string_strip(strings)
output = self.evaluate(output)
self.assertAllEqual(output, [[b"pigs on the wing", b"animals"],
[b"hello", b"world"]])
def test_string_strip_with_empty_strings(self):
strings = [" hello ", "", "world ", " \t \r \n "]
with self.cached_session() as sess:
output = string_ops.string_strip(strings)
output = self.evaluate(output)
self.assertAllEqual(output, [b"hello", b"", b"world", b""])
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/string_to_hash_bucket_op_test.py
+96
View File
@@ -0,0 +1,96 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for StringToHashBucket op from string_ops."""
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import string_ops
from tensorflow.python.platform import test
class StringToHashBucketOpTest(test.TestCase):
@test_util.run_deprecated_v1
def testStringToOneHashBucketFast(self):
with self.cached_session():
input_string = array_ops.placeholder(dtypes.string)
output = string_ops.string_to_hash_bucket_fast(input_string, 1)
result = output.eval(feed_dict={input_string: ['a', 'b', 'c']})
self.assertAllEqual([0, 0, 0], result)
@test_util.run_deprecated_v1
def testStringToHashBucketsFast(self):
with self.cached_session():
input_string = array_ops.placeholder(dtypes.string)
output = string_ops.string_to_hash_bucket_fast(input_string, 10)
result = output.eval(feed_dict={input_string: ['a', 'b', 'c', 'd']})
# Fingerprint64('a') -> 12917804110809363939 -> mod 10 -> 9
# Fingerprint64('b') -> 11795596070477164822 -> mod 10 -> 2
# Fingerprint64('c') -> 11430444447143000872 -> mod 10 -> 2
# Fingerprint64('d') -> 4470636696479570465 -> mod 10 -> 5
self.assertAllEqual([9, 2, 2, 5], result)
@test_util.run_deprecated_v1
def testStringToOneHashBucketLegacyHash(self):
with self.cached_session():
input_string = array_ops.placeholder(dtypes.string)
output = string_ops.string_to_hash_bucket(input_string, 1)
result = output.eval(feed_dict={input_string: ['a', 'b', 'c']})
self.assertAllEqual([0, 0, 0], result)
@test_util.run_deprecated_v1
def testStringToHashBucketsLegacyHash(self):
with self.cached_session():
input_string = array_ops.placeholder(dtypes.string)
output = string_ops.string_to_hash_bucket(input_string, 10)
result = output.eval(feed_dict={input_string: ['a', 'b', 'c']})
# Hash64('a') -> 2996632905371535868 -> mod 10 -> 8
# Hash64('b') -> 5795986006276551370 -> mod 10 -> 0
# Hash64('c') -> 14899841994519054197 -> mod 10 -> 7
self.assertAllEqual([8, 0, 7], result)
def testStringToOneHashBucketStrongOneHashBucket(self):
with self.cached_session():
input_string = constant_op.constant(['a', 'b', 'c'])
output = string_ops.string_to_hash_bucket_strong(
input_string, 1, key=[123, 345])
self.assertAllEqual([0, 0, 0], self.evaluate(output))
def testStringToHashBucketsStrong(self):
with self.cached_session():
input_string = constant_op.constant(['a', 'b', 'c'])
output = string_ops.string_to_hash_bucket_strong(
input_string, 10, key=[98765, 132])
# key = [98765, 132]
# StrongKeyedHash(key, 'a') -> 7157389809176466784 -> mod 10 -> 4
# StrongKeyedHash(key, 'b') -> 15805638358933211562 -> mod 10 -> 2
# StrongKeyedHash(key, 'c') -> 18100027895074076528 -> mod 10 -> 8
self.assertAllEqual([4, 2, 8], self.evaluate(output))
def testStringToHashBucketsStrongInvalidKey(self):
with self.cached_session():
input_string = constant_op.constant(['a', 'b', 'c'])
with self.assertRaisesOpError('Key must have 2 elements'):
string_ops.string_to_hash_bucket_strong(
input_string, 10, key=[98765]).eval()
if __name__ == '__main__':
test.main()
tensorflow/python/kernel_tests/strings_ops/string_to_number_op_test.py
+101
View File
@@ -0,0 +1,101 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for StringToNumber op from parsing_ops."""
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import parsing_ops
from tensorflow.python.platform import test
_ERROR_MESSAGE = "StringToNumberOp could not correctly convert string: "
class StringToNumberOpTest(test.TestCase):
def _test(self, tf_type, good_pairs, bad_pairs):
with self.cached_session():
# Build a small testing graph.
input_string = array_ops.placeholder(dtypes.string)
output = parsing_ops.string_to_number(
input_string, out_type=tf_type)
# Check all the good input/output pairs.
for instr, outnum in good_pairs:
result, = output.eval(feed_dict={input_string: [instr]})
self.assertAllClose([outnum], [result])
# Check that the bad inputs produce the right errors.
for instr, outstr in bad_pairs:
with self.assertRaisesOpError(outstr):
output.eval(feed_dict={input_string: [instr]})
@test_util.run_deprecated_v1
def testToFloat(self):
self._test(dtypes.float32,
[("0", 0), ("3", 3), ("-1", -1),
("1.12", 1.12), ("0xF", 15), (" -10.5", -10.5),
("3.40282e+38", 3.40282e+38),
# Greater than max value of float.
("3.40283e+38", float("INF")),
("-3.40283e+38", float("-INF")),
# Less than min value of float.
("NAN", float("NAN")),
("INF", float("INF"))],
[("10foobar", _ERROR_MESSAGE + "10foobar")])
@test_util.run_deprecated_v1
def testToDouble(self):
self._test(dtypes.float64,
[("0", 0), ("3", 3), ("-1", -1),
("1.12", 1.12), ("0xF", 15), (" -10.5", -10.5),
("3.40282e+38", 3.40282e+38),
# Greater than max value of float.
("3.40283e+38", 3.40283e+38),
# Less than min value of float.
("-3.40283e+38", -3.40283e+38),
("NAN", float("NAN")),
("INF", float("INF"))],
[("10foobar", _ERROR_MESSAGE + "10foobar")])
@test_util.run_deprecated_v1
def testToInt32(self):
self._test(dtypes.int32,
[("0", 0), ("3", 3), ("-1", -1),
(" -10", -10),
("-2147483648", -2147483648),
("2147483647", 2147483647)],
[ # Less than min value of int32.
("-2147483649", _ERROR_MESSAGE + "-2147483649"),
# Greater than max value of int32.
("2147483648", _ERROR_MESSAGE + "2147483648"),
("2.9", _ERROR_MESSAGE + "2.9"),
("10foobar", _ERROR_MESSAGE + "10foobar")])
@test_util.run_deprecated_v1
def testToInt64(self):
self._test(dtypes.int64,
[("0", 0), ("3", 3), ("-1", -1),
(" -10", -10),
("-2147483648", -2147483648),
("2147483647", 2147483647),
("-2147483649", -2147483649), # Less than min value of int32.
("2147483648", 2147483648)], # Greater than max value of int32.
[("2.9", _ERROR_MESSAGE + "2.9"),
("10foobar", _ERROR_MESSAGE + "10foobar")])
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/string_upper_op_test.py
+52
View File
@@ -0,0 +1,52 @@
# -*- coding: utf-8 -*-
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for string_upper_op."""
from tensorflow.python.ops import string_ops
from tensorflow.python.platform import test
class StringUpperOpTest(test.TestCase):
"""Test cases for tf.strings.upper."""
def test_string_upper(self):
strings = ["Pigs on The Wing", "aNimals"]
with self.cached_session():
output = string_ops.string_upper(strings)
output = self.evaluate(output)
self.assertAllEqual(output, [b"PIGS ON THE WING", b"ANIMALS"])
def test_string_upper_2d(self):
strings = [["pigS on THE wIng", "aniMals"], [" hello ", "\n\tWorld! \r \n"]]
with self.cached_session():
output = string_ops.string_upper(strings)
output = self.evaluate(output)
self.assertAllEqual(output, [[b"PIGS ON THE WING", b"ANIMALS"],
[b" HELLO ", b"\n\tWORLD! \r \n"]])
def test_string_upper_unicode(self):
strings = [["óósschloë"]]
with self.cached_session():
output = string_ops.string_upper(strings, encoding="utf-8")
output = self.evaluate(output)
# output: "ÓÓSSCHLOË"
self.assertAllEqual(output, [[b"\xc3\x93\xc3\x93SSCHLO\xc3\x8b"]])
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/substr_op_test.py
+577
View File
@@ -0,0 +1,577 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for Substr op from string_ops."""
from absl.testing import parameterized
import numpy as np
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import test_util
from tensorflow.python.ops import string_ops
from tensorflow.python.platform import test
class SubstrOpTest(test.TestCase, parameterized.TestCase):
@parameterized.parameters(
(np.int32, 1, "BYTE"),
(np.int64, 1, "BYTE"),
(np.int32, -4, "BYTE"),
(np.int64, -4, "BYTE"),
(np.int32, 1, "UTF8_CHAR"),
(np.int64, 1, "UTF8_CHAR"),
(np.int32, -4, "UTF8_CHAR"),
(np.int64, -4, "UTF8_CHAR"),
)
def testScalarString(self, dtype, pos, unit):
test_string = {
"BYTE": b"Hello",
"UTF8_CHAR": u"He\xc3\xc3\U0001f604".encode("utf-8"),
}[unit]
expected_value = {
"BYTE": b"ell",
"UTF8_CHAR": u"e\xc3\xc3".encode("utf-8"),
}[unit]
position = np.array(pos, dtype)
length = np.array(3, dtype)
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
substr = self.evaluate(substr_op)
self.assertAllEqual(substr, expected_value)
@parameterized.parameters(
(np.int32, "BYTE"),
(np.int64, "BYTE"),
(np.int32, "UTF8_CHAR"),
(np.int64, "UTF8_CHAR"),
)
def testScalarString_EdgeCases(self, dtype, unit):
# Empty string
test_string = {
"BYTE": b"",
"UTF8_CHAR": u"".encode("utf-8"),
}[unit]
expected_value = b""
position = np.array(0, dtype)
length = np.array(3, dtype)
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
substr = self.evaluate(substr_op)
self.assertAllEqual(substr, expected_value)
# Full string
test_string = {
"BYTE": b"Hello",
"UTF8_CHAR": u"H\xc3ll\U0001f604".encode("utf-8"),
}[unit]
position = np.array(0, dtype)
length = np.array(5, dtype)
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
substr = self.evaluate(substr_op)
self.assertAllEqual(substr, test_string)
# Full string (Negative)
test_string = {
"BYTE": b"Hello",
"UTF8_CHAR": u"H\xc3ll\U0001f604".encode("utf-8"),
}[unit]
position = np.array(-5, dtype)
length = np.array(5, dtype)
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
substr = self.evaluate(substr_op)
self.assertAllEqual(substr, test_string)
# Length is larger in magnitude than a negative position
test_string = {
"BYTE": b"Hello",
"UTF8_CHAR": u"H\xc3ll\U0001f604".encode("utf-8"),
}[unit]
expected_string = {
"BYTE": b"ello",
"UTF8_CHAR": u"\xc3ll\U0001f604".encode("utf-8"),
}[unit]
position = np.array(-4, dtype)
length = np.array(5, dtype)
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
substr = self.evaluate(substr_op)
self.assertAllEqual(substr, expected_string)
@parameterized.parameters(
(np.int32, 1, "BYTE"),
(np.int64, 1, "BYTE"),
(np.int32, -4, "BYTE"),
(np.int64, -4, "BYTE"),
(np.int32, 1, "UTF8_CHAR"),
(np.int64, 1, "UTF8_CHAR"),
(np.int32, -4, "UTF8_CHAR"),
(np.int64, -4, "UTF8_CHAR"),
)
def testVectorStrings(self, dtype, pos, unit):
test_string = {
"BYTE": [b"Hello", b"World"],
"UTF8_CHAR": [x.encode("utf-8") for x in [u"H\xc3llo",
u"W\U0001f604rld"]],
}[unit]
expected_value = {
"BYTE": [b"ell", b"orl"],
"UTF8_CHAR": [x.encode("utf-8") for x in [u"\xc3ll", u"\U0001f604rl"]],
}[unit]
position = np.array(pos, dtype)
length = np.array(3, dtype)
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
substr = self.evaluate(substr_op)
self.assertAllEqual(substr, expected_value)
@parameterized.parameters(
(np.int32, "BYTE"),
(np.int64, "BYTE"),
(np.int32, "UTF8_CHAR"),
(np.int64, "UTF8_CHAR"),
)
def testMatrixStrings(self, dtype, unit):
test_string = {
"BYTE": [[b"ten", b"eleven", b"twelve"],
[b"thirteen", b"fourteen", b"fifteen"],
[b"sixteen", b"seventeen", b"eighteen"]],
"UTF8_CHAR": [[x.encode("utf-8") for x in [u"\U0001d229\U0001d227n",
u"\xc6\u053c\u025bv\u025bn",
u"tw\u0c1dlv\u025b"]],
[x.encode("utf-8") for x in [u"He\xc3\xc3o",
u"W\U0001f604rld",
u"d\xfcd\xea"]]],
}[unit]
position = np.array(1, dtype)
length = np.array(4, dtype)
expected_value = {
"BYTE": [[b"en", b"leve", b"welv"], [b"hirt", b"ourt", b"ifte"],
[b"ixte", b"even", b"ight"]],
"UTF8_CHAR": [[x.encode("utf-8") for x in [u"\U0001d227n",
u"\u053c\u025bv\u025b",
u"w\u0c1dlv"]],
[x.encode("utf-8") for x in [u"e\xc3\xc3o",
u"\U0001f604rld",
u"\xfcd\xea"]]],
}[unit]
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
substr = self.evaluate(substr_op)
self.assertAllEqual(substr, expected_value)
position = np.array(-3, dtype)
length = np.array(2, dtype)
expected_value = {
"BYTE": [[b"te", b"ve", b"lv"], [b"ee", b"ee", b"ee"],
[b"ee", b"ee", b"ee"]],
"UTF8_CHAR": [[x.encode("utf-8") for x in [u"\U0001d229\U0001d227",
u"v\u025b", u"lv"]],
[x.encode("utf-8") for x in [u"\xc3\xc3", u"rl",
u"\xfcd"]]],
}[unit]
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
substr = self.evaluate(substr_op)
self.assertAllEqual(substr, expected_value)
@parameterized.parameters(
(np.int32, "BYTE"),
(np.int64, "BYTE"),
(np.int32, "UTF8_CHAR"),
(np.int64, "UTF8_CHAR"),
)
def testElementWisePosLen(self, dtype, unit):
test_string = {
"BYTE": [[b"ten", b"eleven", b"twelve"],
[b"thirteen", b"fourteen", b"fifteen"],
[b"sixteen", b"seventeen", b"eighteen"]],
"UTF8_CHAR": [[x.encode("utf-8") for x in [u"\U0001d229\U0001d227n",
u"\xc6\u053c\u025bv\u025bn",
u"tw\u0c1dlv\u025b"]],
[x.encode("utf-8") for x in [u"He\xc3\xc3o",
u"W\U0001f604rld",
u"d\xfcd\xea"]],
[x.encode("utf-8") for x in [u"sixt\xea\xean",
u"se\U00010299enteen",
u"ei\U0001e920h\x86een"]]],
}[unit]
position = np.array([[1, -4, 3], [1, 2, -4], [-5, 2, 3]], dtype)
length = np.array([[2, 2, 4], [4, 3, 2], [5, 5, 5]], dtype)
expected_value = {
"BYTE": [[b"en", b"ev", b"lve"], [b"hirt", b"urt", b"te"],
[b"xteen", b"vente", b"hteen"]],
"UTF8_CHAR": [[x.encode("utf-8") for x in [u"\U0001d227n",
u"\u025bv",
u"lv\u025b"]],
[x.encode("utf-8") for x in [u"e\xc3\xc3o",
u"rld",
u"d\xfc"]],
[x.encode("utf-8") for x in [u"xt\xea\xean",
u"\U00010299ente",
u"h\x86een"]]],
}[unit]
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
substr = self.evaluate(substr_op)
self.assertAllEqual(substr, expected_value)
@parameterized.parameters(
(np.int32, "BYTE"),
(np.int64, "BYTE"),
(np.int32, "UTF8_CHAR"),
(np.int64, "UTF8_CHAR"),
)
def testBroadcast(self, dtype, unit):
# Broadcast pos/len onto input string
test_string = {
"BYTE": [[b"ten", b"eleven", b"twelve"],
[b"thirteen", b"fourteen", b"fifteen"],
[b"sixteen", b"seventeen", b"eighteen"],
[b"nineteen", b"twenty", b"twentyone"]],
"UTF8_CHAR": [[x.encode("utf-8") for x in [u"\U0001d229\U0001d227n",
u"\xc6\u053c\u025bv\u025bn",
u"tw\u0c1dlv\u025b"]],
[x.encode("utf-8") for x in [u"th\xcdrt\xea\xean",
u"f\U0001f604urt\xea\xean",
u"f\xcd\ua09ctee\ua0e4"]],
[x.encode("utf-8") for x in [u"s\xcdxt\xea\xean",
u"se\U00010299enteen",
u"ei\U0001e920h\x86een"]],
[x.encode("utf-8") for x in [u"nineteen",
u"twenty",
u"twentyone"]]],
}[unit]
position = np.array([1, -4, 3], dtype)
length = np.array([1, 2, 3], dtype)
expected_value = {
"BYTE": [[b"e", b"ev", b"lve"], [b"h", b"te", b"tee"],
[b"i", b"te", b"hte"], [b"i", b"en", b"nty"]],
"UTF8_CHAR": [[x.encode("utf-8") for x in [u"\U0001d227",
u"\u025bv", u"lv\u025b"]],
[x.encode("utf-8") for x in [u"h", u"t\xea", u"tee"]],
[x.encode("utf-8") for x in [u"\xcd", u"te", u"h\x86e"]],
[x.encode("utf-8") for x in [u"i", u"en", u"nty"]]],
}[unit]
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
substr = self.evaluate(substr_op)
self.assertAllEqual(substr, expected_value)
# Broadcast input string onto pos/len
test_string = {
"BYTE": [b"thirteen", b"fourteen", b"fifteen"],
"UTF8_CHAR": [x.encode("utf-8") for x in [u"th\xcdrt\xea\xean",
u"f\U0001f604urt\xea\xean",
u"f\xcd\ua09ctee\ua0e4"]],
}[unit]
position = np.array([[1, -2, 3], [-3, 2, 1], [5, 5, -5]], dtype)
length = np.array([[3, 2, 1], [1, 2, 3], [2, 2, 2]], dtype)
expected_value = {
"BYTE": [[b"hir", b"en", b"t"], [b"e", b"ur", b"ift"],
[b"ee", b"ee", b"ft"]],
"UTF8_CHAR": [[x.encode("utf-8") for x in [u"h\xcdr", u"\xean", u"t"]],
[x.encode("utf-8") for x in [u"\xea", u"ur",
u"\xcd\ua09ct"]],
[x.encode("utf-8") for x in [u"\xea\xea", u"\xea\xea",
u"\ua09ct"]]],
}[unit]
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
substr = self.evaluate(substr_op)
self.assertAllEqual(substr, expected_value)
# Test 1D broadcast
test_string = {
"BYTE": b"thirteen",
"UTF8_CHAR": u"th\xcdrt\xea\xean".encode("utf-8"),
}[unit]
position = np.array([1, -4, 7], dtype)
length = np.array([3, 2, 1], dtype)
expected_value = {
"BYTE": [b"hir", b"te", b"n"],
"UTF8_CHAR": [x.encode("utf-8") for x in [u"h\xcdr", u"t\xea", u"n"]],
}[unit]
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
substr = self.evaluate(substr_op)
self.assertAllEqual(substr, expected_value)
@parameterized.parameters(
(np.int32, "BYTE"),
(np.int64, "BYTE"),
(np.int32, "UTF8_CHAR"),
(np.int64, "UTF8_CHAR"),
)
@test_util.run_deprecated_v1
def testBadBroadcast(self, dtype, unit):
test_string = [[b"ten", b"eleven", b"twelve"],
[b"thirteen", b"fourteen", b"fifteen"],
[b"sixteen", b"seventeen", b"eighteen"]]
position = np.array([1, 2, -3, 4], dtype)
length = np.array([1, 2, 3, 4], dtype)
with self.assertRaises(ValueError):
string_ops.substr(test_string, position, length, unit=unit)
@parameterized.parameters(
(np.int32, 6, "BYTE"),
(np.int64, 6, "BYTE"),
(np.int32, -6, "BYTE"),
(np.int64, -6, "BYTE"),
(np.int32, 6, "UTF8_CHAR"),
(np.int64, 6, "UTF8_CHAR"),
(np.int32, -6, "UTF8_CHAR"),
(np.int64, -6, "UTF8_CHAR"),
)
@test_util.run_deprecated_v1
def testOutOfRangeError_Scalar(self, dtype, pos, unit):
# Scalar/Scalar
test_string = {
"BYTE": b"Hello",
"UTF8_CHAR": u"H\xc3ll\U0001f604".encode("utf-8"),
}[unit]
position = np.array(pos, dtype)
length = np.array(3, dtype)
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
with self.assertRaises(errors_impl.InvalidArgumentError):
self.evaluate(substr_op)
@parameterized.parameters(
(np.int32, 4, "BYTE"),
(np.int64, 4, "BYTE"),
(np.int32, -4, "BYTE"),
(np.int64, -4, "BYTE"),
(np.int32, 4, "UTF8_CHAR"),
(np.int64, 4, "UTF8_CHAR"),
(np.int32, -4, "UTF8_CHAR"),
(np.int64, -4, "UTF8_CHAR"),
)
@test_util.run_deprecated_v1
def testOutOfRangeError_VectorScalar(self, dtype, pos, unit):
# Vector/Scalar
test_string = {
"BYTE": [b"good", b"good", b"bad", b"good"],
"UTF8_CHAR": [x.encode("utf-8") for x in [u"g\xc3\xc3d", u"b\xc3d",
u"g\xc3\xc3d"]],
}[unit]
position = np.array(pos, dtype)
length = np.array(1, dtype)
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
with self.assertRaises(errors_impl.InvalidArgumentError):
self.evaluate(substr_op)
@parameterized.parameters(
(np.int32, "BYTE"),
(np.int64, "BYTE"),
(np.int32, "UTF8_CHAR"),
(np.int64, "UTF8_CHAR"),
)
@test_util.run_deprecated_v1
def testOutOfRangeError_MatrixMatrix(self, dtype, unit):
# Matrix/Matrix
test_string = {
"BYTE": [[b"good", b"good", b"good"], [b"good", b"good", b"bad"],
[b"good", b"good", b"good"]],
"UTF8_CHAR": [[x.encode("utf-8") for x in [u"g\xc3\xc3d", u"g\xc3\xc3d",
u"g\xc3\xc3d"]],
[x.encode("utf-8") for x in [u"g\xc3\xc3d", u"g\xc3\xc3d",
u"b\xc3d"]],
[x.encode("utf-8") for x in [u"g\xc3\xc3d", u"g\xc3\xc3d",
u"g\xc3\xc3d"]]],
}[unit]
position = np.array([[1, 2, 3], [1, 2, 4], [1, 2, 3]], dtype)
length = np.array([[3, 2, 1], [1, 2, 3], [2, 2, 2]], dtype)
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
with self.assertRaises(errors_impl.InvalidArgumentError):
self.evaluate(substr_op)
# Matrix/Matrix (with negative)
position = np.array([[1, 2, -3], [1, 2, -4], [1, 2, -3]], dtype)
length = np.array([[3, 2, 1], [1, 2, 3], [2, 2, 2]], dtype)
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
with self.assertRaises(errors_impl.InvalidArgumentError):
self.evaluate(substr_op)
@parameterized.parameters(
(np.int32, "BYTE"),
(np.int64, "BYTE"),
(np.int32, "UTF8_CHAR"),
(np.int64, "UTF8_CHAR"),
)
@test_util.run_deprecated_v1
def testOutOfRangeError_Broadcast(self, dtype, unit):
# Broadcast
test_string = {
"BYTE": [[b"good", b"good", b"good"], [b"good", b"good", b"bad"]],
"UTF8_CHAR": [[x.encode("utf-8") for x in [u"g\xc3\xc3d", u"g\xc3\xc3d",
u"g\xc3\xc3d"]],
[x.encode("utf-8") for x in [u"g\xc3\xc3d", u"g\xc3\xc3d",
u"b\xc3d"]]],
}[unit]
position = np.array([1, 2, 4], dtype)
length = np.array([1, 2, 3], dtype)
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
with self.assertRaises(errors_impl.InvalidArgumentError):
self.evaluate(substr_op)
# Broadcast (with negative)
position = np.array([-1, -2, -4], dtype)
length = np.array([1, 2, 3], dtype)
substr_op = string_ops.substr(test_string, position, length, unit=unit)
with self.cached_session():
with self.assertRaises(errors_impl.InvalidArgumentError):
self.evaluate(substr_op)
@parameterized.parameters(
(np.int32, "BYTE"),
(np.int64, "BYTE"),
(np.int32, "UTF8_CHAR"),
(np.int64, "UTF8_CHAR"),
)
@test_util.run_deprecated_v1
def testMismatchPosLenShapes(self, dtype, unit):
test_string = {
"BYTE": [[b"ten", b"eleven", b"twelve"],
[b"thirteen", b"fourteen", b"fifteen"],
[b"sixteen", b"seventeen", b"eighteen"]],
"UTF8_CHAR": [[x.encode("utf-8") for x in [u"\U0001d229\U0001d227n",
u"\xc6\u053c\u025bv\u025bn",
u"tw\u0c1dlv\u025b"]],
[x.encode("utf-8") for x in [u"th\xcdrt\xea\xean",
u"f\U0001f604urt\xea\xean",
u"f\xcd\ua09ctee\ua0e4"]],
[x.encode("utf-8") for x in [u"s\xcdxt\xea\xean",
u"se\U00010299enteen",
u"ei\U0001e920h\x86een"]]],
}[unit]
position = np.array([[1, 2, 3]], dtype)
length = np.array([2, 3, 4], dtype)
# Should fail: position/length have different rank
with self.assertRaises(ValueError):
string_ops.substr(test_string, position, length)
position = np.array([[1, 2, 3], [1, 2, 3], [1, 2, 3]], dtype)
length = np.array([[2, 3, 4]], dtype)
# Should fail: position/length have different dimensionality
with self.assertRaises(ValueError):
string_ops.substr(test_string, position, length)
@test_util.run_deprecated_v1
def testWrongDtype(self):
with self.cached_session():
with self.assertRaises(TypeError):
string_ops.substr(b"test", 3.0, 1)
with self.assertRaises(TypeError):
string_ops.substr(b"test", 3, 1.0)
@test_util.run_deprecated_v1
def testInvalidUnit(self):
with self.cached_session():
with self.assertRaises(ValueError):
string_ops.substr(b"test", 3, 1, unit="UTF8")
def testInvalidPos(self):
# Test case for GitHub issue 46900.
with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)):
x = string_ops.substr(b"abc", len=1, pos=[1, -1])
self.evaluate(x)
with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)):
x = string_ops.substr(b"abc", len=1, pos=[1, 2])
self.evaluate(x)
@parameterized.parameters(
(np.int32, "BYTE", 1),
(np.int32, "BYTE", 2),
(np.int64, "BYTE", 1),
(np.int64, "BYTE", 2),
(np.int32, "UTF8_CHAR", 1),
(np.int32, "UTF8_CHAR", 2),
(np.int64, "UTF8_CHAR", 1),
(np.int64, "UTF8_CHAR", 2),
)
def testSingleString(self, dtype, unit, rank):
test_string = {
"BYTE": [b"abcdefghijklmnopqrstuvwxyz"],
"UTF8_CHAR": [
u"\U0001d229\U0001d227n\U0001d229\U0001d227n\U0001d229\U0001d227n"
.encode("utf-8")
],
}[unit]
position = np.array([1, 2, 3], dtype)
length = np.array([1, 2, 1], dtype)
expected_value = {
"BYTE": [b"b", b"cd", b"d"],
"UTF8_CHAR": [
x.decode("utf-8") for x in
[b"\xf0\x9d\x88\xa7", b"n\xf0\x9d\x88\xa9", b"\xf0\x9d\x88\xa9"]
],
}[unit]
test_string_tensor = np.array(test_string)
expected_string_tensor = np.array(expected_value)
if rank == 2:
test_string_tensor = np.expand_dims(test_string_tensor, axis=0)
expected_string_tensor = np.expand_dims(expected_string_tensor, axis=0)
substr_op = string_ops.substr(
test_string_tensor, position, length, unit=unit)
with self.cached_session():
substr = self.evaluate(substr_op)
self.assertAllEqual(substr, expected_string_tensor)
self.assertEqual(substr.ndim, rank)
@parameterized.parameters(
(np.int32, "BYTE", 3),
(np.int32, "BYTE", 10),
(np.int64, "BYTE", 3),
(np.int64, "BYTE", 10),
(np.int32, "UTF8_CHAR", 3),
(np.int32, "UTF8_CHAR", 10),
(np.int64, "UTF8_CHAR", 3),
(np.int64, "UTF8_CHAR", 10),
)
def testSingleStringHighRankFails(self, dtype, unit, rank):
test_string = {
"BYTE": [b"abcdefghijklmnopqrstuvwxyz"],
"UTF8_CHAR": [
u"\U0001d229\U0001d227n\U0001d229\U0001d227n\U0001d229\U0001d227n"
.encode("utf-8")
],
}[unit]
position = np.array([1, 2, 3], dtype)
length = np.array([1, 2, 1], dtype)
test_string_tensor = np.array(test_string)
for _ in range(rank - 1):
test_string_tensor = np.expand_dims(test_string_tensor, axis=0)
with self.assertRaises(errors_impl.UnimplementedError):
# substr is only supported up to rank 2
substr_op = string_ops.substr(
test_string_tensor, position, length, unit=unit)
self.evaluate(substr_op)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/unicode_decode_op_test.py
+747
View File
@@ -0,0 +1,747 @@
# -*- coding: utf-8 -*-
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for unicode_decode and unicode_decode_with_splits."""
from absl.testing import parameterized
import numpy as np
from tensorflow.python.eager import context
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import sparse_tensor
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_string_ops
from tensorflow.python.ops.ragged import ragged_factory_ops
from tensorflow.python.ops.ragged import ragged_string_ops
from tensorflow.python.platform import test
def _nested_encode(x, encoding):
"""Encode each string in a nested list with `encoding`."""
if isinstance(x, list):
return [_nested_encode(v, encoding) for v in x]
else:
return x.encode(encoding)
def _nested_codepoints(x):
"""Replace each string in a nested list with a list of its codepoints."""
# Works for Python 2 and 3, and for both UCS2 and UCS4 builds
if isinstance(x, list):
return [_nested_codepoints(v) for v in x]
else:
b = list(x.encode("utf-32-be"))
if any(isinstance(c, str) for c in b):
b = [ord(c) for c in b]
return [(b0 << 24) + (b1 << 16) + (b2 << 8) + b3
for b0, b1, b2, b3 in zip(b[::4], b[1::4], b[2::4], b[3::4])]
def _nested_offsets(x, encoding):
"""Replace each string in a nested list with a list of start offsets."""
if isinstance(x, list):
return [_nested_offsets(v, encoding) for v in x]
else:
if not x:
return []
encoded_x = x.encode("utf-32-be")
encoded_chars = [encoded_x[i:i + 4] for i in range(0, len(encoded_x), 4)]
char_lens = [
len(c.decode("utf-32-be").encode(encoding)) for c in encoded_chars
]
return [0] + np.cumsum(char_lens).tolist()[:-1]
def _nested_splitchars(x, encoding):
"""Replace each string in a nested list with a list of char substrings."""
if isinstance(x, list):
return [_nested_splitchars(v, encoding) for v in x]
else:
b = x.encode("utf-32-be")
chars = zip(b[::4], b[1::4], b[2::4], b[3::4])
if str is bytes:
return [b"".join(c).decode("utf-32-be").encode(encoding) for c in chars]
else:
return [bytes(c).decode("utf-32-be").encode(encoding) for c in chars]
def _make_sparse_tensor(indices, values, dense_shape, dtype=np.int32):
return sparse_tensor.SparseTensorValue(
np.array(indices, np.int64), np.array(values, dtype),
np.array(dense_shape, np.int64))
@test_util.run_all_in_graph_and_eager_modes
class UnicodeDecodeTest(test_util.TensorFlowTestCase,
parameterized.TestCase):
def testScalarDecode(self):
text = constant_op.constant(u"仅今年前".encode("utf-8"))
chars = ragged_string_ops.unicode_decode(text, "utf-8")
self.assertAllEqual(chars, [ord(c) for c in u"仅今年前"])
def testScalarDecodeWithOffset(self):
text = constant_op.constant(u"仅今年前".encode("utf-8"))
chars, starts = ragged_string_ops.unicode_decode_with_offsets(text, "utf-8")
self.assertAllEqual(chars, [ord(c) for c in u"仅今年前"])
self.assertAllEqual(starts, [0, 3, 6, 9])
def testVectorDecode(self):
text = constant_op.constant([u"仅今年前".encode("utf-8"), b"hello"])
chars = ragged_string_ops.unicode_decode(text, "utf-8")
expected_chars = [[ord(c) for c in u"仅今年前"],
[ord(c) for c in u"hello"]]
self.assertAllEqual(chars, expected_chars)
def testVectorDecodeWithOffset(self):
text = constant_op.constant([u"仅今年前".encode("utf-8"), b"hello"])
chars, starts = ragged_string_ops.unicode_decode_with_offsets(text, "utf-8")
expected_chars = [[ord(c) for c in u"仅今年前"],
[ord(c) for c in u"hello"]]
self.assertAllEqual(chars, expected_chars)
self.assertAllEqual(starts, [[0, 3, 6, 9], [0, 1, 2, 3, 4]])
@parameterized.parameters([
{"texts": u"仅今年前"},
{"texts": [u"G\xf6\xf6dnight", u"\U0001f60a"]},
{"texts": ["Hello", "world", "", u"👍"]},
{"texts": [["Hi", "there"], ["", u"\U0001f60a"]], "ragged_rank": 0},
{"texts": [["Hi", "there", ""], [u"😊"]], "ragged_rank": 1},
{"texts": [[[u"😊", u"🤠🧐"], []], [[u"🤓👻🤖"]]], "ragged_rank": 2},
{"texts": [[[u"😊"], [u"🤠🧐"]], [[u"🤓👻🤖"]]], "ragged_rank": 1},
{"texts": [[[u"😊"], [u"🤠🧐"]], [[u"🤓"], [u"👻"]]], "ragged_rank": 0},
{"texts": []}
]) # pyformat: disable
def testBasicDecode(self, texts, ragged_rank=None):
input_tensor = ragged_factory_ops.constant_value(
_nested_encode(texts, "UTF-8"), ragged_rank=ragged_rank, dtype=bytes)
result = ragged_string_ops.unicode_decode(input_tensor, "UTF-8")
expected = _nested_codepoints(texts)
self.assertAllEqual(expected, result)
@parameterized.parameters([
{"texts": u"仅今年前"},
{"texts": [u"G\xf6\xf6dnight", u"\U0001f60a"]},
{"texts": ["Hello", "world", "", u"👍"]},
{"texts": [["Hi", "there"], ["", u"\U0001f60a"]], "ragged_rank": 0},
{"texts": [["Hi", "there", ""], [u"😊"]], "ragged_rank": 1},
{"texts": [[[u"😊", u"🤠🧐"], []], [[u"🤓👻🤖"]]], "ragged_rank": 2},
{"texts": []}
]) # pyformat: disable
def testBasicDecodeWithOffsets(self, texts, ragged_rank=None):
input_tensor = ragged_factory_ops.constant_value(
_nested_encode(texts, "UTF-8"), ragged_rank=ragged_rank, dtype=bytes)
result = ragged_string_ops.unicode_decode_with_offsets(
input_tensor, "UTF-8")
expected_codepoints = _nested_codepoints(texts)
expected_offsets = _nested_offsets(texts, "UTF-8")
self.assertAllEqual(expected_codepoints, result[0])
self.assertAllEqual(expected_offsets, result[1])
def testDocstringExamples(self):
texts = [s.encode("utf8") for s in [u"G\xf6\xf6dnight", u"\U0001f60a"]]
codepoints1 = ragged_string_ops.unicode_decode(texts, "UTF-8")
codepoints2, offsets = ragged_string_ops.unicode_decode_with_offsets(
texts, "UTF-8")
self.assertAllEqual(
codepoints1, [[71, 246, 246, 100, 110, 105, 103, 104, 116], [128522]])
self.assertAllEqual(
codepoints2, [[71, 246, 246, 100, 110, 105, 103, 104, 116], [128522]])
self.assertAllEqual(offsets, [[0, 1, 3, 5, 6, 7, 8, 9, 10], [0]])
@parameterized.parameters([
dict(
texts=["Hello", "world", "", u"👍"],
expected=_make_sparse_tensor(
indices=[[0, 0], [0, 1], [0, 2], [0, 3], [0, 4], [1, 0], [1, 1],
[1, 2], [1, 3], [1, 4], [3, 0]],
values=[72, 101, 108, 108, 111, 119, 111, 114, 108, 100, 128077],
dense_shape=[4, 5])),
dict(
texts=[["Hi", "there"], ["", u"\U0001f60a"]],
expected=_make_sparse_tensor(
indices=[[0, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 1], [0, 1, 2],
[0, 1, 3], [0, 1, 4], [1, 1, 0]],
values=[72, 105, 116, 104, 101, 114, 101, 128522],
dense_shape=[2, 2, 5])),
dict(
texts=[],
expected=_make_sparse_tensor(np.zeros([0, 2], np.int64), [], [0, 0])),
])
def testDecodeWithSparseOutput(self, texts, expected):
input_tensor = np.array(_nested_encode(texts, "UTF-8"), dtype=bytes)
result = ragged_string_ops.unicode_decode(input_tensor, "UTF-8").to_sparse()
self.assertIsInstance(result, sparse_tensor.SparseTensor)
self.assertAllEqual(expected.indices, result.indices)
self.assertAllEqual(expected.values, result.values)
self.assertAllEqual(expected.dense_shape, result.dense_shape)
@parameterized.parameters([
dict(
texts=["Hello", "world", "", u"👍"],
expected=[[72, 101, 108, 108, 111], [119, 111, 114, 108, 100],
[-1, -1, -1, -1, -1], [0x1F44D, -1, -1, -1, -1]]),
dict(
texts=[["Hi", "there"], ["", u"\U0001f60a"]],
expected=[[[72, 105, -1, -1, -1], [116, 104, 101, 114, 101]],
[[-1, -1, -1, -1, -1], [128522, -1, -1, -1, -1]]],
ragged_rank=0),
dict(
texts=[["Hi", "there", ""], [u"😊"]],
expected=[[[72, 105, -1, -1, -1],
[116, 104, 101, 114, 101],
[-1, -1, -1, -1, -1]],
[[128522, -1, -1, -1, -1],
[-1, -1, -1, -1, -1],
[-1, -1, -1, -1, -1]]]),
dict(
texts=[[[u"😊", u"🤠🧐"], []], [[u"🤓👻🤖"]]],
expected=[
[[[128522, -1, -1], [129312, 129488, -1]],
[[-1, -1, -1], [-1, -1, -1]]],
[[[129299, 128123, 129302], [-1, -1, -1]],
[[-1, -1, -1], [-1, -1, -1]]]]),
dict(texts=[], expected=np.zeros([0, 0], np.int64)),
]) # pyformat: disable
def testDecodeWithPaddedOutput(self, texts, expected, ragged_rank=None):
input_tensor = ragged_factory_ops.constant_value(
_nested_encode(texts, "UTF-8"), ragged_rank=ragged_rank, dtype=bytes)
result = ragged_string_ops.unicode_decode(
input_tensor, "UTF-8").to_tensor(default_value=-1)
self.assertAllEqual(expected, result)
@parameterized.parameters([
dict(
input=[b"\xFE", b"hello", b"==\xFF==", b"world"],
input_encoding="UTF-8",
errors="replace",
expected=[[0xFFFD],
[ord('h'), ord('e'), ord('l'), ord('l'), ord('o')],
[ord('='), ord('='), 0xFFFD, ord('='), ord('=')],
[ord('w'), ord('o'), ord('r'), ord('l'), ord('d')]]),
dict(
input=[b"\xFE", b"hello", b"==\xFF==", b"world"],
input_encoding="UTF-8",
errors="replace",
replacement_char=0,
expected=[[0], [ord('h'), ord('e'), ord('l'), ord('l'), ord('o')],
[ord('='), ord('='), 0, ord('='), ord('=')],
[ord('w'), ord('o'), ord('r'), ord('l'), ord('d')]]),
dict(
input=[b"\xFE", b"hello", b"==\xFF==", b"world"],
input_encoding="UTF-8",
errors="ignore",
expected=[[], [ord('h'), ord('e'), ord('l'), ord('l'), ord('o')],
[ord('='), ord('='), ord('='), ord('=')],
[ord('w'), ord('o'), ord('r'), ord('l'), ord('d')]]),
dict(
input=[b"\x00", b"hello", b"==\x01==", b"world",
# C1 control characters are not replaced.
u"\x80\x9f".encode()],
input_encoding="UTF-8",
replace_control_characters=True,
expected=[[0xFFFD],
[ord('h'), ord('e'), ord('l'), ord('l'), ord('o')],
[61, 61, 65533, 61, 61],
[ord('w'), ord('o'), ord('r'), ord('l'), ord('d')],
[0x80, 0x9F]]),
dict(
input=[b"\x00", b"hello", b"==\x01==", b"world",
u"\u0080\u009f".encode()],
input_encoding="UTF-8",
replace_control_characters=True,
replacement_char=0,
expected=[[0], [ord('h'), ord('e'), ord('l'), ord('l'), ord('o')],
[ord('='), ord('='), 0, ord('='), ord('=')],
[ord('w'), ord('o'), ord('r'), ord('l'), ord('d')],
[0x80, 0x9F]]),
dict(
input=[b"\xed\xa0\xbd", # single high surrogate
b"\xed\xb8\x80", # single low surrogate
b"\xed\xa0\xbd\xed\xb8\x80"], # surrogate pair
input_encoding="UTF-8",
expected=[[0xFFFD, 0xFFFD, 0xFFFD],
[0xFFFD, 0xFFFD, 0xFFFD],
[0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD]]),
dict(
input=["\ufdd0".encode(), # noncharacter
"\ufffe\uffff".encode(), # last two in BMP
"\U0010ffff".encode(), # last in plane 16 = last in Unicode
b"\xc0\x80", # overlong U+0000 encoding
b"\xf4\x90\x80\x80"], # U+110000, beyond Unicode
input_encoding="UTF-8",
expected=[[0xFDD0],
[0xFFFE, 0xFFFF],
[0x10FFFF],
[0xFFFD] * 2,
[0xFFFD] * 4]),
]) # pyformat: disable
def testErrorModes(self, expected=None, **args):
result = ragged_string_ops.unicode_decode(**args)
self.assertAllEqual(expected, result)
@parameterized.parameters([
dict(
input=[b"\xFE", b"hello", b"==\xFF==", b"world"],
input_encoding="UTF-8",
errors="replace",
expected=[[0xFFFD],
[ord('h'), ord('e'), ord('l'), ord('l'), ord('o')],
[ord('='), ord('='), 0xFFFD, ord('='), ord('=')],
[ord('w'), ord('o'), ord('r'), ord('l'), ord('d')]],
expected_offsets=[[0], [0, 1, 2, 3, 4],
[0, 1, 2, 3, 4], [0, 1, 2, 3, 4]]),
dict(
input=[b"\xFE", b"hello", b"==\xFF==", b"world"],
input_encoding="UTF-8",
errors="replace",
replacement_char=0,
expected=[[0], [ord('h'), ord('e'), ord('l'), ord('l'), ord('o')],
[ord('='), ord('='), 0, ord('='), ord('=')],
[ord('w'), ord('o'), ord('r'), ord('l'), ord('d')]],
expected_offsets=[[0], [0, 1, 2, 3, 4],
[0, 1, 2, 3, 4], [0, 1, 2, 3, 4]]),
dict(
input=[b"\xFE", b"hello", b"==\xFF==", b"world"],
input_encoding="UTF-8",
errors="ignore",
expected=[[], [ord('h'), ord('e'), ord('l'), ord('l'), ord('o')],
[ord('='), ord('='), ord('='), ord('=')],
[ord('w'), ord('o'), ord('r'), ord('l'), ord('d')]],
expected_offsets=[[], [0, 1, 2, 3, 4],
[0, 1, 3, 4], [0, 1, 2, 3, 4]]),
dict(
input=[b"\x00", b"hello", b"==\x01==", b"world"],
input_encoding="UTF-8",
replace_control_characters=True,
expected=[[0xFFFD],
[ord('h'), ord('e'), ord('l'), ord('l'), ord('o')],
[ord('='), ord('='), 0xFFFD, ord('='), ord('=')],
[ord('w'), ord('o'), ord('r'), ord('l'), ord('d')]],
expected_offsets=[[0], [0, 1, 2, 3, 4],
[0, 1, 2, 3, 4], [0, 1, 2, 3, 4]]),
dict(
input=[b"\x00", b"hello", b"==\x01==", b"world"],
input_encoding="UTF-8",
replace_control_characters=True,
replacement_char=0,
expected=[[0], [ord('h'), ord('e'), ord('l'), ord('l'), ord('o')],
[0x3D, 0x3D, 0, 0x3D, 0x3D],
[ord('w'), ord('o'), ord('r'), ord('l'), ord('d')]],
expected_offsets=[[0], [0, 1, 2, 3, 4],
[0, 1, 2, 3, 4], [0, 1, 2, 3, 4]]),
dict(
input=[b"\xD8\x01"],
input_encoding="UTF-8",
replacement_char=0x41,
expected=[[0x41, 1]],
expected_offsets=[[0, 1]]),
]) # pyformat: disable
def testErrorModesWithOffsets(self,
expected=None,
expected_offsets=None,
**args):
result = ragged_string_ops.unicode_decode_with_offsets(**args)
self.assertAllEqual(result[0], expected)
self.assertAllEqual(result[1], expected_offsets)
@parameterized.parameters(
("UTF-8", [u"こんにちは", u"你好", u"Hello"]),
("UTF-16-BE", [u"こんにちは", u"你好", u"Hello"]),
("UTF-32-BE", [u"こんにちは", u"你好", u"Hello"]),
("US-ASCII", [u"Hello", "world"]),
("ISO-8859-1", [u"ÀÈÓ", "AEO"]),
("SHIFT-JIS", [u"Hello", u"こんにちは"]),
)
def testDecodeWithDifferentEncodings(self, encoding, texts):
expected = _nested_codepoints(texts)
input_tensor = constant_op.constant(_nested_encode(texts, encoding))
result = ragged_string_ops.unicode_decode(input_tensor, encoding)
self.assertAllEqual(expected, result)
@parameterized.parameters(
("UTF-8", [u"こんにちは", u"你好", u"Hello"]),
("UTF-16-BE", [u"こんにちは", u"你好", u"Hello"]),
("UTF-32-BE", [u"こんにちは", u"你好", u"Hello"]),
("US-ASCII", [u"Hello", "world"]),
("ISO-8859-1", [u"ÀÈÓ", "AEO"]),
("SHIFT-JIS", [u"Hello", u"こんにちは"]),
)
def testDecodeWithOffsetsWithDifferentEncodings(self, encoding, texts):
expected_codepoints = _nested_codepoints(texts)
expected_offsets = _nested_offsets(texts, encoding)
input_tensor = constant_op.constant(_nested_encode(texts, encoding))
result = ragged_string_ops.unicode_decode_with_offsets(
input_tensor, encoding)
self.assertAllEqual(expected_codepoints, result[0])
self.assertAllEqual(expected_offsets, result[1])
@parameterized.parameters([
dict(input=[b"\xFEED"],
errors="strict",
input_encoding="UTF-8",
exception=errors.InvalidArgumentError,
message="Invalid formatting on input string"),
dict(input="x",
input_encoding="UTF-8",
replacement_char=11141111,
exception=errors.InvalidArgumentError,
message="replacement_char out of unicode codepoint range"),
dict(input="x",
input_encoding="UTF-8",
errors="oranguatan",
exception=(ValueError, errors.InvalidArgumentError)),
]) # pyformat: disable
def testExceptions(self, exception=None, message=None, **args):
with self.assertRaisesRegex(exception, message):
self.evaluate(ragged_string_ops.unicode_decode(**args))
def testUnknownRankError(self):
if context.executing_eagerly():
return
s = array_ops.placeholder(dtypes.string)
message = "Rank of `input` must be statically known."
with self.assertRaisesRegex(ValueError, message):
self.evaluate(ragged_string_ops.unicode_decode(s, input_encoding="UTF-8"))
@parameterized.parameters([
dict(
doc="Single string",
input=_nested_encode([u"仅今年前"], "utf-8"),
input_encoding="UTF-8",
expected_char_values=_nested_codepoints(u"仅今年前"),
expected_row_splits=[0, 4],
expected_char_to_byte_starts=[0, 3, 6, 9]),
dict(
doc="Multiple strings",
input=_nested_encode([u"仅今年前", u"你好"], "utf-8"),
input_encoding="UTF-8",
expected_char_values=_nested_codepoints(u"仅今年前你好"),
expected_row_splits=[0, 4, 6],
expected_char_to_byte_starts=[0, 3, 6, 9, 0, 3]),
dict(
doc="errors=replace",
input=b"=\xFE=",
input_encoding="UTF-8",
errors="replace",
expected_char_values=[0x3D, 0xFFFD, 0x3D],
expected_row_splits=[0, 3],
expected_char_to_byte_starts=[0, 1, 2]),
dict(
doc="errors=ignore",
input=b"=\xFE=",
input_encoding="UTF-8",
errors="ignore",
expected_char_values=[61, 61],
expected_row_splits=[0, 2],
expected_char_to_byte_starts=[0, 2]),
])
def testDecodeGenOp(self,
doc,
expected_row_splits=None,
expected_char_values=None,
expected_char_to_byte_starts=None,
**args):
"""Test for the c++ interface (gen_string_ops.unicode_decode)."""
result = gen_string_ops.unicode_decode_with_offsets(**args)
self.assertAllEqual(expected_row_splits, result.row_splits)
self.assertAllEqual(expected_char_values, result.char_values)
self.assertAllEqual(expected_char_to_byte_starts,
result.char_to_byte_starts)
@test_util.run_all_in_graph_and_eager_modes
class UnicodeSplitTest(test_util.TensorFlowTestCase,
parameterized.TestCase):
def testScalarSplit(self):
text = constant_op.constant(u"仅今年前".encode("UTF-8"))
chars = ragged_string_ops.unicode_split(text, "UTF-8")
self.assertAllEqual(chars, [c.encode("UTF-8") for c in u"仅今年前"])
def testScalarSplitWithOffset(self):
text = constant_op.constant(u"仅今年前".encode("UTF-8"))
chars, starts = ragged_string_ops.unicode_split_with_offsets(text, "UTF-8")
self.assertAllEqual(chars, [c.encode("UTF-8") for c in u"仅今年前"])
self.assertAllEqual(starts, [0, 3, 6, 9])
def testVectorSplit(self):
text = constant_op.constant([u"仅今年前".encode("UTF-8"), b"hello"])
chars = ragged_string_ops.unicode_split(text, "UTF-8")
expected_chars = [[c.encode("UTF-8") for c in u"仅今年前"],
[c.encode("UTF-8") for c in u"hello"]]
self.assertAllEqual(chars, expected_chars)
def testVectorSplitWithOffset(self):
text = constant_op.constant([u"仅今年前".encode("UTF-8"), b"hello"])
chars, starts = ragged_string_ops.unicode_split_with_offsets(text, "UTF-8")
expected_chars = [[c.encode("UTF-8") for c in u"仅今年前"],
[c.encode("UTF-8") for c in u"hello"]]
self.assertAllEqual(chars, expected_chars)
self.assertAllEqual(starts, [[0, 3, 6, 9], [0, 1, 2, 3, 4]])
@parameterized.parameters([
{"texts": u"仅今年前"},
{"texts": [u"G\xf6\xf6dnight", u"\U0001f60a"]},
{"texts": ["Hello", "world", "", u"👍"]},
{"texts": [["Hi", "there"], ["", u"\U0001f60a"]], "ragged_rank": 0},
{"texts": [["Hi", "there", ""], [u"😊"]], "ragged_rank": 1},
{"texts": [[[u"😊", u"🤠🧐"], []], [[u"🤓👻🤖"]]], "ragged_rank": 2},
{"texts": []}
]) # pyformat: disable
def testBasicSplit(self, texts, ragged_rank=None):
input_tensor = ragged_factory_ops.constant_value(
_nested_encode(texts, "UTF-8"), ragged_rank=ragged_rank, dtype=bytes)
result = ragged_string_ops.unicode_split(input_tensor, "UTF-8")
expected = _nested_splitchars(texts, "UTF-8")
self.assertAllEqual(expected, result)
@parameterized.parameters([
{"texts": u"仅今年前"},
{"texts": [u"G\xf6\xf6dnight", u"\U0001f60a"]},
{"texts": ["Hello", "world", "", u"👍"]},
{"texts": [["Hi", "there"], ["", u"\U0001f60a"]], "ragged_rank": 0},
{"texts": [["Hi", "there", ""], [u"😊"]], "ragged_rank": 1},
{"texts": [[[u"😊", u"🤠🧐"], []], [[u"🤓👻🤖"]]], "ragged_rank": 2},
{"texts": []}
]) # pyformat: disable
def testBasicSplitWithOffsets(self, texts, ragged_rank=None):
input_tensor = ragged_factory_ops.constant_value(
_nested_encode(texts, "UTF-8"), ragged_rank=ragged_rank, dtype=bytes)
result = ragged_string_ops.unicode_split_with_offsets(input_tensor, "UTF-8")
expected_codepoints = _nested_splitchars(texts, "UTF-8")
expected_offsets = _nested_offsets(texts, "UTF-8")
self.assertAllEqual(expected_codepoints, result[0])
self.assertAllEqual(expected_offsets, result[1])
def testDocstringExamples(self):
texts = [s.encode("utf8") for s in [u"G\xf6\xf6dnight", u"\U0001f60a"]]
codepoints1 = ragged_string_ops.unicode_split(texts, "UTF-8")
codepoints2, offsets = ragged_string_ops.unicode_split_with_offsets(
texts, "UTF-8")
self.assertAllEqual(
codepoints1,
[[b"G", b"\xc3\xb6", b"\xc3\xb6", b"d", b"n", b"i", b"g", b"h", b"t"],
[b"\xf0\x9f\x98\x8a"]])
self.assertAllEqual(
codepoints2,
[[b"G", b"\xc3\xb6", b"\xc3\xb6", b"d", b"n", b"i", b"g", b"h", b"t"],
[b"\xf0\x9f\x98\x8a"]])
self.assertAllEqual(offsets, [[0, 1, 3, 5, 6, 7, 8, 9, 10], [0]])
@parameterized.parameters([
dict(
texts=["Hello", "world", "", u"👍"],
expected=_make_sparse_tensor(
indices=[[0, 0], [0, 1], [0, 2], [0, 3], [0, 4], [1, 0], [1, 1],
[1, 2], [1, 3], [1, 4], [3, 0]],
values=[b"H", b"e", b"l", b"l", b"o",
b"w", b"o", b"r", b"l", b"d", b"\xf0\x9f\x91\x8d"],
dense_shape=[4, 5],
dtype=bytes)),
dict(
texts=[["Hi", "there"], ["", u"\U0001f60a"]],
expected=_make_sparse_tensor(
indices=[[0, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 1], [0, 1, 2],
[0, 1, 3], [0, 1, 4], [1, 1, 0]],
values=[b"H", b"i", b"t", b"h", b"e", b"r", b"e",
b"\xf0\x9f\x98\x8a"],
dense_shape=[2, 2, 5],
dtype=bytes)),
dict(
texts=[],
expected=_make_sparse_tensor(
np.zeros([0, 2], np.int64), [], [0, 0], dtype=bytes)),
]) # pyformat: disable
def testSplitWithSparseOutput(self, texts, expected):
input_tensor = np.array(_nested_encode(texts, "UTF-8"), dtype=bytes)
result = ragged_string_ops.unicode_split(input_tensor, "UTF-8").to_sparse()
self.assertIsInstance(result, sparse_tensor.SparseTensor)
self.assertAllEqual(expected.indices, result.indices)
self.assertAllEqual(expected.values, result.values)
self.assertAllEqual(expected.dense_shape, result.dense_shape)
@parameterized.parameters([
dict(
texts=["Hello", "world", "", u"👍"],
expected=[[b"H", b"e", b"l", b"l", b"o"],
[b"w", b"o", b"r", b"l", b"d"],
["", "", "", "", ""],
[b"\xf0\x9f\x91\x8d", "", "", "", ""]]),
dict(
texts=[["Hi", "there"], ["", u"\U0001f60a"]],
expected=[[[b"H", b"i", "", "", ""],
[b"t", b"h", b"e", b"r", b"e"]],
[["", "", "", "", ""],
[b"\xf0\x9f\x98\x8a", "", "", "", ""]]],
ragged_rank=0),
dict(
texts=[["Hi", "there", ""], [u"😊"]],
expected=[[[b"H", b"i", "", "", ""],
[b"t", b"h", b"e", b"r", b"e"],
["", "", "", "", ""]],
[[b"\xf0\x9f\x98\x8a", "", "", "", ""],
["", "", "", "", ""],
["", "", "", "", ""]]]),
dict(
texts=[[[u"😊", u"🤠🧐"], []], [[u"🤓👻🤖"]]],
expected=[[[[b"\xf0\x9f\x98\x8a", "", ""],
[b"\xf0\x9f\xa4\xa0", b"\xf0\x9f\xa7\x90", ""]],
[["", "", ""],
["", "", ""]]],
[[[b"\xf0\x9f\xa4\x93", b"\xf0\x9f\x91\xbb",
b"\xf0\x9f\xa4\x96"],
["", "", ""]],
[["", "", ""],
["", "", ""]]]]),
dict(texts=[], expected=np.zeros([0, 0], np.int64)),
]) # pyformat: disable
def testSplitWithPaddedOutput(self, texts, expected, ragged_rank=None):
input_tensor = ragged_factory_ops.constant_value(
_nested_encode(texts, "UTF-8"), ragged_rank=ragged_rank, dtype=bytes)
result = ragged_string_ops.unicode_split(
input_tensor, "UTF-8").to_tensor(default_value="")
self.assertAllEqual(np.array(expected, dtype=bytes), result)
@parameterized.parameters([
dict(
input=[b"\xFE", b"hello", b"==\xFF==", b"world"],
input_encoding="UTF-8",
errors="replace",
expected=[[b"\xef\xbf\xbd"],
[b"h", b"e", b"l", b"l", b"o"],
[b"=", b"=", b"\xef\xbf\xbd", b"=", b"="],
[b"w", b"o", b"r", b"l", b"d"]]),
dict(
input=[b"\xFE", b"hello", b"==\xFF==", b"world"],
input_encoding="UTF-8",
errors="replace",
replacement_char=0,
expected=[[b"\x00"],
[b"h", b"e", b"l", b"l", b"o"],
[b"=", b"=", b"\x00", b"=", b"="],
[b"w", b"o", b"r", b"l", b"d"]]),
dict(
input=[b"\xFE", b"hello", b"==\xFF==", b"world"],
input_encoding="UTF-8",
errors="ignore",
expected=[[],
[b"h", b"e", b"l", b"l", b"o"],
[b"=", b"=", b"=", b"="],
[b"w", b"o", b"r", b"l", b"d"]]),
]) # pyformat: disable
def testErrorModes(self, expected=None, **args):
result = ragged_string_ops.unicode_split(**args)
self.assertAllEqual(expected, result)
@parameterized.parameters([
dict(
input=[b"\xFE", b"hello", b"==\xFF==", b"world"],
input_encoding="UTF-8",
errors="replace",
expected=[[b"\xef\xbf\xbd"],
[b"h", b"e", b"l", b"l", b"o"],
[b"=", b"=", b"\xef\xbf\xbd", b"=", b"="],
[b"w", b"o", b"r", b"l", b"d"]],
expected_offsets=[[0], [0, 1, 2, 3, 4],
[0, 1, 2, 3, 4], [0, 1, 2, 3, 4]]),
dict(
input=[b"\xFE", b"hello", b"==\xFF==", b"world"],
input_encoding="UTF-8",
errors="replace",
replacement_char=0,
expected=[[b"\x00"],
[b"h", b"e", b"l", b"l", b"o"],
[b"=", b"=", b"\x00", b"=", b"="],
[b"w", b"o", b"r", b"l", b"d"]],
expected_offsets=[[0], [0, 1, 2, 3, 4],
[0, 1, 2, 3, 4], [0, 1, 2, 3, 4]]),
dict(
input=[b"\xFE", b"hello", b"==\xFF==", b"world"],
input_encoding="UTF-8",
errors="ignore",
expected=[[],
[b"h", b"e", b"l", b"l", b"o"],
[b"=", b"=", b"=", b"="],
[b"w", b"o", b"r", b"l", b"d"]],
expected_offsets=[[], [0, 1, 2, 3, 4],
[0, 1, 3, 4], [0, 1, 2, 3, 4]]),
]) # pyformat: disable
def testErrorModesWithOffsets(self,
expected=None,
expected_offsets=None,
**args):
result = ragged_string_ops.unicode_split_with_offsets(**args)
self.assertAllEqual(expected, result[0])
self.assertAllEqual(expected_offsets, result[1])
@parameterized.parameters(
("UTF-8", [u"こんにちは", u"你好", u"Hello"]),
("UTF-16-BE", [u"こんにちは", u"你好", u"Hello"]),
("UTF-32-BE", [u"こんにちは", u"你好", u"Hello"]),
)
def testSplitWithDifferentEncodings(self, encoding, texts):
expected = _nested_splitchars(texts, encoding)
input_tensor = constant_op.constant(_nested_encode(texts, encoding))
result = ragged_string_ops.unicode_split(input_tensor, encoding)
self.assertAllEqual(expected, result)
@parameterized.parameters(
("UTF-8", [u"こんにちは", u"你好", u"Hello"]),
("UTF-16-BE", [u"こんにちは", u"你好", u"Hello"]),
("UTF-32-BE", [u"こんにちは", u"你好", u"Hello"]),
)
def testSplitWithOffsetsWithDifferentEncodings(self, encoding, texts):
expected_codepoints = _nested_splitchars(texts, encoding)
expected_offsets = _nested_offsets(texts, encoding)
input_tensor = constant_op.constant(_nested_encode(texts, encoding))
result = ragged_string_ops.unicode_split_with_offsets(
input_tensor, encoding)
self.assertAllEqual(expected_codepoints, result[0])
self.assertAllEqual(expected_offsets, result[1])
@parameterized.parameters([
dict(input=[b"\xFEED"],
errors="strict",
input_encoding="UTF-8",
exception=errors.InvalidArgumentError,
message="Invalid formatting on input string"),
dict(input="x",
input_encoding="UTF-8",
replacement_char=11141111,
exception=errors.InvalidArgumentError,
message="replacement_char out of unicode codepoint range"),
dict(input="x",
input_encoding="UTF-8",
errors="oranguatan",
exception=(ValueError, errors.InvalidArgumentError)),
]) # pyformat: disable
def testExceptions(self, exception=None, message=None, **args):
with self.assertRaisesRegex(exception, message):
self.evaluate(ragged_string_ops.unicode_split(**args))
def testUnknownRankError(self):
if context.executing_eagerly():
return
s = array_ops.placeholder(dtypes.string)
message = "Rank of `input` must be statically known."
with self.assertRaisesRegex(ValueError, message):
self.evaluate(ragged_string_ops.unicode_decode(s, input_encoding="UTF-8"))
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/unicode_encode_op_test.py
+358
View File
@@ -0,0 +1,358 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for UnicodeEncode op from ragged_string_ops."""
from absl.testing import parameterized
import numpy as np
from tensorflow.python.eager import def_function
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import errors_impl as errors
from tensorflow.python.framework import tensor_spec
from tensorflow.python.framework import test_util
from tensorflow.python.ops.ragged import ragged_factory_ops
from tensorflow.python.ops.ragged import ragged_string_ops
from tensorflow.python.ops.ragged import ragged_tensor
from tensorflow.python.ops.ragged import ragged_tensor_value
from tensorflow.python.platform import test
class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
def assertAllEqual(self, rt, expected):
with self.cached_session() as sess:
value = sess.run(rt)
if isinstance(value, np.ndarray):
value = value.tolist()
elif isinstance(value, ragged_tensor_value.RaggedTensorValue):
value = value.to_list()
self.assertEqual(value, expected)
def testScalar(self):
with self.cached_session():
with self.assertRaises(ValueError):
ragged_string_ops.unicode_encode(72, "UTF-8")
with self.cached_session():
with self.assertRaises(ValueError):
ragged_string_ops.unicode_encode(constant_op.constant(72), "UTF-8")
def testRequireParams(self):
with self.cached_session():
with self.assertRaises(TypeError):
ragged_string_ops.unicode_encode() # pylint: disable=no-value-for-parameter
with self.cached_session():
with self.assertRaises(TypeError):
ragged_string_ops.unicode_encode(72) # pylint: disable=no-value-for-parameter
with self.cached_session():
with self.assertRaises(TypeError):
ragged_string_ops.unicode_encode(encoding="UTF-8") # pylint: disable=no-value-for-parameter,unexpected-keyword-arg
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
def testStrictErrorsValid(self, encoding):
# Noncharacters are not errors.
# See https://www.unicode.org/versions/corrigendum9.html.
test_value = np.array([ord("H"), ord("e"), ord("o"), 0, ord("\b"), 0x1F600,
0xFDD0, # noncharacter
0xFFFE, 0xFFFF, # last two in BMP
0x10FFFF], # largest valid
np.int32)
expected_value = (
u"Heo\0\b\U0001f600\ufdd0\ufffe\uffff\U0010ffff"
).encode(encoding)
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
"strict")
self.assertAllEqual(unicode_encode_op, expected_value)
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
def testStrictErrorsNegative(self, encoding):
test_value = np.array([-1], np.int32)
with self.cached_session() as session:
with self.assertRaises(errors.InvalidArgumentError):
session.run(
ragged_string_ops.unicode_encode(test_value, encoding, "strict"))
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
def testStrictErrorsTooLarge(self, encoding):
test_value = np.array([0x110000], np.int32)
with self.cached_session() as session:
with self.assertRaises(errors.InvalidArgumentError):
session.run(
ragged_string_ops.unicode_encode(test_value, encoding, "strict"))
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
def testStrictErrorsSurrogatePair(self, encoding):
test_value = np.array([0xD83D, 0xDE00], np.int32)
with self.cached_session() as session:
with self.assertRaises(errors.InvalidArgumentError):
session.run(
ragged_string_ops.unicode_encode(test_value, encoding, "strict"))
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
@test_util.run_v1_only("b/120545219")
def testIgnoreErrors(self, encoding):
# Noncharacters are not errors.
# See https://www.unicode.org/versions/corrigendum9.html.
test_value = np.array([ord('H'), ord('e'), 0x7FFFFFFF, -1, ord('o'),
0, ord('\b'), 0x1F600, # valid
0xD83D, 0xDE00, # invalid, surrogate code points
0xFDD0, # noncharacter
0xFFFE, 0xFFFF, # last two in BMP
0x10FFFF, # last in plane 16 = last in Unicode
0x110000], # invalid, beyond Unicode
np.int32)
expected_value = (
u"Heo\0\b\U0001F600\ufdd0\ufffe\uffff\U0010ffff"
).encode(encoding)
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
"ignore")
self.assertAllEqual(unicode_encode_op, expected_value)
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
@test_util.run_v1_only("b/120545219")
def testIgnoreErrorsSurrogate(self, encoding):
test_value = np.array([0xD83D, 0xDE00], np.int32)
expected_value = b""
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
"ignore")
self.assertAllEqual(unicode_encode_op, expected_value)
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
@test_util.run_v1_only("b/120545219")
def testReplaceErrors(self, encoding):
# Noncharacters are not errors.
# See https://www.unicode.org/versions/corrigendum9.html.
test_value = np.array([ord('H'), ord('e'), 0x7FFFFFFF, -1, ord('o'),
0, ord('\b'), 0x1F600, # valid
0xD83D, 0xDE00, # invalid, surrogate code points
0xFDD0, # noncharacter
0xFFFE, 0xFFFF, # last two in BMP
0x10FFFF, # last in plane 16 = last in Unicode
0x110000], # invalid, beyond Unicode
np.int32)
expected_value = (u"He\ufffd\ufffdo\0\b\U0001F600\ufffd\ufffd"
u"\ufdd0\ufffe\uffff\U0010ffff\ufffd").encode(encoding)
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
"replace")
self.assertAllEqual(unicode_encode_op, expected_value)
# Test custom replacement character
test_value = np.array([ord('H'), ord('e'), 0x7FFFFFFF, -1, ord('o')],
np.int32)
expected_value = u"Heooo".encode(encoding)
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
"replace", ord('o'))
self.assertAllEqual(unicode_encode_op, expected_value)
# Verify "replace" is default
test_value = np.array([ord('H'), ord('e'), 0x7FFFFFFF, -1, ord('o')],
np.int32)
expected_value = u"He\U0000fffd\U0000fffdo".encode(encoding)
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
self.assertAllEqual(unicode_encode_op, expected_value)
# Verify non-default replacement with an unpaired surrogate.
test_value = np.array([0xD801], np.int32)
expected_value = "\u0041".encode(encoding)
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
"replace", 0x41)
self.assertAllEqual(unicode_encode_op, expected_value)
# Test with a noncharacter code point. These are not errors.
# See https://www.unicode.org/versions/corrigendum9.html.
test_value = np.array([0x1FFFF], np.int32)
expected_value = u"\U0001ffff".encode(encoding)
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
"replace", 0x41)
self.assertAllEqual(unicode_encode_op, expected_value)
# Replacement_char must be within range
test_value = np.array([ord('H'), ord('e'), 0x7FFFFFFF, -1, ord('o')],
np.int32)
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
"replace", 0x110000)
with self.assertRaises(errors.InvalidArgumentError):
self.evaluate(unicode_encode_op)
# -- regular Tensor tests -- #
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
@test_util.run_v1_only("b/120545219")
def testVector(self, encoding):
test_value = np.array([ord('H'), ord('e'), ord('l'), ord('l'), ord('o')],
np.int32)
expected_value = u"Hello".encode(encoding)
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
self.assertAllEqual(unicode_encode_op, expected_value)
test_value = np.array([ord('H'), ord('e'), 0xC3, 0xC3, 0x1F604], np.int32)
expected_value = u"He\xc3\xc3\U0001f604".encode(encoding)
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
self.assertAllEqual(unicode_encode_op, expected_value)
# Single character string
test_value = np.array([ord('H')], np.int32)
expected_value = u"H".encode(encoding)
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
self.assertAllEqual(unicode_encode_op, expected_value)
test_value = np.array([0x1F604], np.int32)
expected_value = u"\U0001f604".encode(encoding)
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
self.assertAllEqual(unicode_encode_op, expected_value)
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
@test_util.run_v1_only("b/120545219")
def testMatrix(self, encoding):
test_value = np.array(
[[72, 0x1F604, 108, 108, 111], [87, 0x1F604, 114, 108, 100]], np.int32)
expected_value = [
u"H\U0001f604llo".encode(encoding), u"W\U0001f604rld".encode(encoding)
]
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
self.assertAllEqual(unicode_encode_op, expected_value)
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
@test_util.run_v1_only("b/120545219")
def test3DimMatrix(self, encoding):
test_value = constant_op.constant(
[[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100]],
[[102, 105, 120, 101, 100], [119, 111, 114, 100, 115]],
[[72, 121, 112, 101, 114], [99, 117, 98, 101, 46]]], np.int32)
expected_value = [[u"Hello".encode(encoding), u"World".encode(encoding)],
[u"fixed".encode(encoding), u"words".encode(encoding)],
[u"Hyper".encode(encoding), u"cube.".encode(encoding)]]
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
self.assertAllEqual(unicode_encode_op, expected_value)
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
@test_util.run_v1_only("b/120545219")
def test4DimMatrix(self, encoding):
test_value = constant_op.constant(
[[[[72, 101, 108, 108, 111]], [[87, 111, 114, 108, 100]]],
[[[102, 105, 120, 101, 100]], [[119, 111, 114, 100, 115]]],
[[[72, 121, 112, 101, 114]], [[99, 117, 98, 101, 46]]]], np.int32)
expected_value = [[[u"Hello".encode(encoding)],
[u"World".encode(encoding)]],
[[u"fixed".encode(encoding)],
[u"words".encode(encoding)]],
[[u"Hyper".encode(encoding)],
[u"cube.".encode(encoding)]]]
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
self.assertAllEqual(unicode_encode_op, expected_value)
# -- Ragged Tensor tests -- #
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
@test_util.run_v1_only("b/120545219")
def testRaggedMatrix(self, encoding):
test_value = ragged_factory_ops.constant(
[[ord('H'), 0xC3, ord('l'), ord('l'), ord('o')],
[ord('W'), 0x1F604, ord('r'), ord('l'), ord('d'), ord('.')]], np.int32)
expected_value = [
u"H\xc3llo".encode(encoding), u"W\U0001f604rld.".encode(encoding)
]
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
self.assertAllEqual(unicode_encode_op, expected_value)
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
@test_util.run_v1_only("b/120545219")
def test3DimMatrixWithRagged2ndDim(self, encoding):
test_value = ragged_factory_ops.constant(
[[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100]],
[[102, 105, 120, 101, 100]],
[[72, 121, 112, 101, 114], [119, 111, 114, 100, 115],
[99, 117, 98, 101, 46]]], np.int32)
expected_value = [[u"Hello".encode(encoding), u"World".encode(encoding)],
[u"fixed".encode(encoding)],
[
u"Hyper".encode(encoding), u"words".encode(encoding),
u"cube.".encode(encoding)
]]
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
self.assertAllEqual(unicode_encode_op, expected_value)
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
@test_util.run_v1_only("b/120545219")
def test3DimMatrixWithRagged3rdDim(self, encoding):
test_value = ragged_factory_ops.constant(
[[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100, 46]],
[[68, 111, 110, 39, 116], [119, 195, 114, 114, 121, 44, 32, 98, 101]],
[[0x1F604], []]], np.int32)
expected_value = [[u"Hello".encode(encoding), u"World.".encode(encoding)],
[
u"Don't".encode(encoding),
u"w\xc3rry, be".encode(encoding)
], [u"\U0001f604".encode(encoding), u"".encode(encoding)]]
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
self.assertAllEqual(unicode_encode_op, expected_value)
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
@test_util.run_v1_only("b/120545219")
def test3DimMatrixWithRagged2ndAnd3rdDim(self, encoding):
test_value = ragged_factory_ops.constant(
[[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100, 46]], [],
[[0x1F604]]], np.int32)
expected_value = [[u"Hello".encode(encoding), u"World.".encode(encoding)],
[], [u"\U0001f604".encode(encoding)]]
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
self.assertAllEqual(unicode_encode_op, expected_value)
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
@test_util.run_v1_only("b/120545219")
def test4DimRaggedMatrix(self, encoding):
test_value = ragged_factory_ops.constant(
[[[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100]]],
[[[]], [[72, 121, 112, 101]]]], np.int32)
expected_value = [[[u"Hello".encode(encoding), u"World".encode(encoding)]],
[[u"".encode(encoding)], [u"Hype".encode(encoding)]]]
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
self.assertAllEqual(unicode_encode_op, expected_value)
@parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
@test_util.run_v1_only("b/120545219")
def testRaggedMatrixWithMultiDimensionInnerValues(self, encoding):
test_flat_values = constant_op.constant([[[72, 101, 108, 108, 111],
[87, 111, 114, 108, 100]],
[[102, 105, 120, 101, 100],
[119, 111, 114, 100, 115]],
[[72, 121, 112, 101, 114],
[99, 117, 98, 101, 46]]])
test_row_splits = [
constant_op.constant([0, 2, 3], dtype=np.int64),
constant_op.constant([0, 1, 1, 3], dtype=np.int64)
]
test_value = ragged_tensor.RaggedTensor.from_nested_row_splits(
test_flat_values, test_row_splits)
expected_value = [[[[u"Hello".encode(encoding), u"World".encode(encoding)]],
[]],
[[[u"fixed".encode(encoding), u"words".encode(encoding)],
[u"Hyper".encode(encoding),
u"cube.".encode(encoding)]]]]
unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
self.assertAllEqual(unicode_encode_op, expected_value)
def testUnknownInputRankError(self):
# Use a tf.function that erases shape information.
@def_function.function(input_signature=[tensor_spec.TensorSpec(None)])
def f(v):
return ragged_string_ops.unicode_encode(v, "UTF-8")
with self.assertRaisesRegex(
ValueError, "Rank of input_tensor must be statically known."):
f([72, 101, 108, 108, 111])
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/unicode_script_op_test.py
+84
View File
@@ -0,0 +1,84 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#===============================================================================
"""Functional tests for UnicodeScript op."""
from tensorflow.python.client import session
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import string_ops
from tensorflow.python.platform import benchmark
from tensorflow.python.platform import test
class UnicodeScriptOpTest(test.TestCase):
@test_util.run_deprecated_v1
def testValidScripts(self):
inputs = [
ord("a"),
0x0411, # CYRILLIC CAPITAL LETTER BE
0x82b8, # CJK UNIFIED IDEOGRAPH-82B8
ord(",")
]
with self.cached_session():
input_vector = constant_op.constant(inputs, dtypes.int32)
outputs = string_ops.unicode_script(input_vector).eval()
self.assertAllEqual(
outputs,
[
25, # USCRIPT_LATIN (LATN)
8, # USCRIPT_CYRILLIC (CYRL)
17, # USCRIPT_HAN (HANI)
0 # USCRIPT_COMMON (ZYYY)
])
@test_util.run_deprecated_v1
def testInvalidScript(self):
inputs = [-100, 0xffffff]
with self.cached_session():
input_vector = constant_op.constant(inputs, dtypes.int32)
outputs = string_ops.unicode_script(input_vector).eval()
self.assertAllEqual(outputs, [-1, -1])
class UnicodeScriptBenchmarks(test.Benchmark):
# Generate some random-ish input by jumping around in unicode characters
def _generateBenchmarkInput(self, size):
chars = []
i = 0
offset = 0
continuity_size = 20
while i < size:
chars.append(ord("a") + offset)
i += 1
offset += 1
if i % continuity_size == 0:
offset += 100
if offset > 0x1F940:
offset = 0
return chars
def benchmark_unicode_script(self):
with session.Session(config=benchmark.benchmark_config()) as sess:
chars = self._generateBenchmarkInput(1000000)
script = string_ops.unicode_script(chars)
self.run_op_benchmark(sess, script.op, min_iters=100)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/unicode_transcode_op_test.py
+439
View File
@@ -0,0 +1,439 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for unicode_transcode op."""
from absl.testing import parameterized
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops import string_ops
from tensorflow.python.platform import test
# Note: for now only tests for algorithmic converters since no file-based
# converters can be loaded. TODO(gbillock): add ability to include at least
# the ucmcore converters from the conversion data sets.
class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
def test_transcode_utf8_simple(self):
strings = [[b"a", b"abc"], [b"ABC", b"DEF"]]
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
strings,
input_encoding="UTF-8",
output_encoding="UTF-8",
errors="replace",
replacement_char=ord(" "),
replace_control_characters=False)
values = self.evaluate(outputs)
self.assertAllEqual(values, strings)
outputs = string_ops.unicode_transcode(
strings,
input_encoding="ISO-8859-1",
output_encoding="UTF-8",
errors="replace",
replacement_char=ord(" "),
replace_control_characters=False)
values = self.evaluate(outputs)
self.assertAllEqual(values, strings)
outputs = string_ops.unicode_transcode(
strings,
input_encoding="US-ASCII",
output_encoding="UTF-8",
errors="replace",
replacement_char=ord(" "),
replace_control_characters=False)
values = self.evaluate(outputs)
self.assertAllEqual(values, strings)
def test_transcode_utf16_to_utf8(self):
strings = [b"\x00a\x00b\x20\xAC", b"\xD8\x01\xDC\x37"] # U+10437
expected = [s.decode("UTF-16-BE").encode("UTF-8") for s in strings]
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
strings,
input_encoding="UTF-16",
output_encoding="UTF-8",
errors="replace",
replacement_char=ord(" "),
replace_control_characters=False)
values = self.evaluate(outputs)
self.assertAllEqual(values, expected)
def test_transcode_bad_utf8(self):
bad_string = b"\x00\xff"
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
bad_string,
input_encoding="UTF-8",
output_encoding="UTF-8",
errors="replace",
replacement_char=ord(" "),
replace_control_characters=True)
values = self.evaluate(outputs)
self.assertAllEqual(values, b" ")
outputs = string_ops.unicode_transcode(
bad_string,
input_encoding="UTF-8",
output_encoding="UTF-8",
errors="replace",
replacement_char=ord(" "),
replace_control_characters=False)
values = self.evaluate(outputs)
self.assertAllEqual(values, b"\x00 ")
def test_transcode_bad_utf8_with_some_good(self):
bad_string = b"abc\xffabcdefg"
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
bad_string,
input_encoding="UTF-8",
output_encoding="UTF-8",
errors="replace",
replacement_char=ord(" "),
replace_control_characters=False)
values = self.evaluate(outputs)
self.assertAllEqual(values, b"abc abcdefg")
def test_transcode_bad_utf8_with_defaults(self):
bad_string = b"\x00\xff"
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
bad_string, input_encoding="UTF-8", output_encoding="UTF-8")
values = self.evaluate(outputs)
self.assertAllEqual(values, b"\x00\xef\xbf\xbd")
def test_transcode_bad_utf8_with_space_replacement(self):
bad_string = b"\x00\xff"
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
bad_string, input_encoding="UTF-8", output_encoding="UTF-8",
replacement_char=ord(" "))
values = self.evaluate(outputs)
self.assertAllEqual(values, b"\x00 ")
@test_util.run_deprecated_v1
def test_transcode_bad_utf8_with_strict_errors(self):
bad_string = b"\x00\xff"
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
bad_string,
input_encoding="UTF-8",
output_encoding="UTF-8",
errors="strict")
with self.assertRaisesOpError(
"Invalid formatting on input string"):
self.evaluate(outputs)
@test_util.run_deprecated_v1
def test_transcode_bad_utf8_start_with_strict_errors(self):
bad_string = b"\xffabcd"
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
bad_string,
input_encoding="UTF-8",
output_encoding="UTF-8",
errors="strict")
with self.assertRaisesOpError(
"Invalid formatting on input string"):
self.evaluate(outputs)
def test_transcode_bad_utf8_with_elision_of_malformatting(self):
bad_string = b"\x00\xff"
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
bad_string,
input_encoding="UTF-8",
output_encoding="UTF-8",
errors="ignore")
values = self.evaluate(outputs)
self.assertAllEqual(values, b"\x00")
def test_transcode_bad_utf8_with_elision_including_control_chars(self):
bad_string = b"\x00\xff"
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
bad_string,
input_encoding="UTF-8",
output_encoding="UTF-8",
errors="ignore",
replace_control_characters=True)
values = self.evaluate(outputs)
self.assertAllEqual(values, b"")
def test_transcode_bad_utf8_termination_with_defaults(self):
bad_string = b"a\xf0"
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
bad_string, input_encoding="UTF-8", output_encoding="UTF-8")
values = self.evaluate(outputs)
self.assertAllEqual(values, b"a\xef\xbf\xbd") # 0xFFFD
def test_transcode_utf8_with_replacement_char(self):
strings = [b"a\xef\xbf\xbd"]
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
strings, input_encoding="UTF-8", output_encoding="UTF-8",
errors="strict")
values = self.evaluate(outputs)
self.assertAllEqual(values, [b"a\xef\xbf\xbd"])
outputs = string_ops.unicode_transcode(
strings, input_encoding="UTF-8", output_encoding="UTF-8",
errors="replace", replacement_char=ord("?"))
values = self.evaluate(outputs)
self.assertAllEqual(values, [b"a\xef\xbf\xbd"])
def test_transcode_utf8_to_utf16(self):
strings = [b"ab\xe2\x82\xac", b"\xf0\x90\x90\xb7"] # U+10437
expected = [s.decode("UTF-8").encode("UTF-16-BE") for s in strings]
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
strings,
input_encoding="UTF-8",
output_encoding="UTF-16-BE",
replacement_char=ord(" "),
replace_control_characters=False)
values = self.evaluate(outputs)
self.assertAllEqual(values, expected)
def test_transcode_utf32_to_utf8(self):
strings = [
b"\x00\x00\x00a\x00\x00\x00b\x00\x00\x20\xAC", b"\x00\x01\x04\x37"
] # U+10437
expected = [s.decode("UTF-32-BE").encode("UTF-8") for s in strings]
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
strings,
input_encoding="UTF-32",
output_encoding="UTF-8",
replacement_char=ord(" "),
replace_control_characters=False)
values = self.evaluate(outputs)
self.assertAllEqual(values, expected)
def test_transcode_utf8_to_utf32(self):
strings = [b"ab\xe2\x82\xac", b"\xf0\x90\x90\xb7"]
expected = [s.decode("UTF-8").encode("UTF-32-BE") for s in strings]
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
strings,
input_encoding="UTF-8",
output_encoding="UTF-32-BE",
replacement_char=ord(" "),
replace_control_characters=False)
values = self.evaluate(outputs)
self.assertAllEqual(values, expected)
# Documentation in ICU suggests that getNextUChar may produce a different
# error code if the input sequence contains particular non-coding sequences.
# This test checks that condition.
def test_transcode_ascii_with_shift_chars(self):
strings = [b"\x0e\x0e", b"\x0f\x0f"]
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
strings,
input_encoding="US-ASCII",
output_encoding="UTF-8",
replacement_char=ord(" "),
replace_control_characters=False)
values = self.evaluate(outputs)
self.assertAllEqual(values, strings)
def test_transcode_utf8_with_bom(self):
bom_string = b"\xef\xbb\xbfabcdefg"
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
bom_string, input_encoding="UTF-8", output_encoding="UTF-8")
values = self.evaluate(outputs)
self.assertAllEqual(values, b"\xef\xbb\xbfabcdefg") # BOM preserved
outputs = string_ops.unicode_transcode(
bom_string, input_encoding="UTF-8", output_encoding="UTF-16-BE")
values = self.evaluate(outputs)
utf16expected = bom_string.decode("UTF-8").encode("UTF-16-BE")
self.assertAllEqual(values, utf16expected)
def test_transcode_utf16_le_be_with_bom(self):
bom_string = b"\xfe\xff\x00\x61" # Big-endian BOM with 'a' encoded
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
bom_string, input_encoding="UTF-16-BE", output_encoding="UTF-8")
values = self.evaluate(outputs)
# BOM is preserved in output
self.assertAllEqual(values, b"\xef\xbb\xbfa")
outputs = string_ops.unicode_transcode(
bom_string, input_encoding="UTF-16-LE", output_encoding="UTF-8")
values = self.evaluate(outputs)
# mangled BOM and value from (incorrect) LE encoding
self.assertAllEqual(values, b"\xef\xbf\xbe\xe6\x84\x80")
bom_string = b"\xff\xfe\x61\x00" # Little-endian BOM with 'a' encoded
outputs = string_ops.unicode_transcode(
bom_string, input_encoding="UTF-16-LE", output_encoding="UTF-8")
values = self.evaluate(outputs)
self.assertAllEqual(values, b"\xef\xbb\xbfa")
@parameterized.parameters(
# BOM is stripped if it is used to decide the byte order of the input.
(b"\xfe\xff\x00*", "UTF-16", b"*"),
(b"\xff\xfe*\x00", "UTF-16", b"*"),
# BOM is *not* stripped if it is not used to decide the byte order of
# the input.
(b"\xef\xbb\xbf*", "UTF-8", b"\xef\xbb\xbf*"),
(b"\xfe\xff\x00*", "UTF-16-BE", b"\xef\xbb\xbf*"),
(b"\xff\xfe*\x00", "UTF-16-LE", b"\xef\xbb\xbf*"),
# If the encoding is UTF-16, and no BOM is present, then UTF-16-BE
# is assumed.
(b"\x00*", "UTF-16", b"*"),
# BOM is never stripped from any position other than the beginning of
# the string, for any encoding.
(b"<\xef\xbb\xbf>", "UTF-8", b"<\xef\xbb\xbf>"),
(b"\x00<\xfe\xff\x00>", "UTF-16", b"<\xef\xbb\xbf>"),
(b"\x00<\xfe\xff\x00>", "UTF-16-BE", b"<\xef\xbb\xbf>"),
(b"<\x00\xff\xfe>\x00", "UTF-16-LE", b"<\xef\xbb\xbf>"),
(b"\xfe\xff\x00<\xfe\xff\x00>", "UTF-16", b"<\xef\xbb\xbf>"),
(b"\xff\xfe<\x00\xff\xfe>\x00", "UTF-16", b"<\xef\xbb\xbf>"),
)
@test_util.run_deprecated_v1
def test_bom_handling(self, string, input_encoding, expected):
with self.test_session():
output = string_ops.unicode_transcode(
string, input_encoding=input_encoding, output_encoding="UTF-8")
self.assertAllEqual(output, expected)
@test_util.run_deprecated_v1
def test_invalid_encoding_causes_errors(self):
strings = [[b"a", b"abc"], [b"ABC", b"DEF"]]
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
strings,
input_encoding="invalid",
output_encoding="UTF-8",
errors="replace",
replacement_char=ord(" "),
replace_control_characters=False)
with self.assertRaisesOpError(
"Could not create converter for input encoding: invalid"):
self.evaluate(outputs)
with self.assertRaisesRegex(ValueError, "Op passed string 'invalid'"):
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
strings,
input_encoding="UTF-8",
output_encoding="invalid",
errors="replace",
replacement_char=ord(" "),
replace_control_characters=False)
self.evaluate(outputs)
@test_util.run_deprecated_v1
def test_invalid_error_policy_causes_errors(self):
strings = [[b"a", b"abc"], [b"ABC", b"DEF"]]
with self.assertRaisesRegex(
ValueError, "'invalid' not in: \"strict\", \"replace\", \"ignore\"."):
with self.cached_session() as sess:
outputs = string_ops.unicode_transcode(
strings,
input_encoding="UTF-8",
output_encoding="UTF-8",
errors="invalid",
replacement_char=ord(" "),
replace_control_characters=False)
self.evaluate(outputs)
def test_forwarding(self):
with self.cached_session():
# Generate an input that is uniquely consumed by the transcode op.
# This exercises code paths which are optimized for this case
# (e.g., using forwarding).
inp = string_ops.substr(
constant_op.constant([b"AbCdEfG", b"HiJkLmN"], dtypes.string),
pos=0,
len=5)
transcoded = string_ops.unicode_transcode(
inp, input_encoding="UTF-8", output_encoding="UTF-8")
self.assertAllEqual([b"AbCdE", b"HiJkL"], transcoded)
@test_util.run_deprecated_v1
def test_cjk_encodings(self):
strings_ja = [
b"\x5c\x5c", # Yen sign
b"\x8f\x70", # kanji character "waza"
b"\x83\x4f"
] # katakana character "gu"
strings_zh_cn = [b"\xca\xf5"] # simplified "shu4"
strings_zh_tw = [b"\xb3\x4e"] # traditional "shu4"
strings_ko = [b"\xc7\xd1\xb9\xce"] # hangul "hanmin"
expected_ja = [s.decode("shift_jis").encode("UTF-8") for s in strings_ja]
expected_zh_cn = [
s.decode("gb18030").encode("UTF-8") for s in strings_zh_cn
]
expected_zh_tw = [s.decode("big5").encode("UTF-8") for s in strings_zh_tw]
expected_ko = [s.decode("euc_kr").encode("UTF-8") for s in strings_ko]
with self.cached_session() as sess:
outputs_ja = string_ops.unicode_transcode(
strings_ja,
input_encoding="shift_jis",
output_encoding="UTF-8",
replacement_char=ord(" "),
replace_control_characters=False)
outputs_zh_cn = string_ops.unicode_transcode(
strings_zh_cn,
input_encoding="gb18030",
output_encoding="UTF-8",
replacement_char=ord(" "),
replace_control_characters=False)
outputs_zh_tw = string_ops.unicode_transcode(
strings_zh_tw,
input_encoding="big5",
output_encoding="UTF-8",
replacement_char=ord(" "),
replace_control_characters=False)
outputs_ko = string_ops.unicode_transcode(
strings_ko,
input_encoding="euc_kr",
output_encoding="UTF-8",
replacement_char=ord(" "),
replace_control_characters=False)
result_ja, result_zh_cn, result_zh_tw, result_ko = sess.run(
[outputs_ja, outputs_zh_cn, outputs_zh_tw, outputs_ko])
self.assertAllEqual(result_ja, expected_ja)
self.assertAllEqual(result_zh_cn, expected_zh_cn)
self.assertAllEqual(result_zh_tw, expected_zh_tw)
self.assertAllEqual(result_ko, expected_ko)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/strings_ops/unsorted_segment_join_op_test.py
+303
View File
@@ -0,0 +1,303 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for unsorted_segment_join_op."""
from absl.testing import parameterized
import numpy as np
from tensorflow.python.eager import context
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import test_util
from tensorflow.python.ops import string_ops
from tensorflow.python.platform import test
class UnicodeTestCase(test.TestCase):
"""Test case with Python3-compatible string comparator."""
def assertAllEqualUnicode(self, truth, actual):
self.assertAllEqual(
np.array(truth).astype('U'),
np.array(actual).astype('U'))
@test_util.run_all_in_graph_and_eager_modes
class UnsortedSegmentJoinOpTest(UnicodeTestCase, parameterized.TestCase):
def test_basic_np_array(self):
inputs = [['Y', 'q', 'c'], ['Y', '6', '6'], ['p', 'G', 'a']]
segment_ids = [1, 0, 1]
num_segments = 2
separator = ':'
output_array = [['Y', '6', '6'], ['Y:p', 'q:G', 'c:a']]
res = self.evaluate(
string_ops.unsorted_segment_join(
inputs=inputs,
segment_ids=segment_ids,
num_segments=num_segments,
separator=separator))
self.assertAllEqual(res.shape, np.array(output_array).shape)
self.assertAllEqualUnicode(res, output_array)
def test_segment_id_and_input_empty(self):
inputs = np.array([], dtype=np.bytes_)
segment_ids = np.array([], dtype=np.int32)
num_segments = 3
separator = ':'
output_array = ['', '', '']
res = self.evaluate(
string_ops.unsorted_segment_join(
inputs=inputs,
segment_ids=segment_ids,
num_segments=num_segments,
separator=separator))
self.assertAllEqual(res.shape, np.array(output_array).shape)
self.assertAllEqualUnicode(res, output_array)
def test_type_check(self):
inputs = [['Y', 'q', 'c'], ['Y', '6', '6'], ['p', 'G', 'a']]
segment_ids = np.array([1, 0, 1], dtype=np.int32)
num_segments = np.array(2, dtype=np.int32)
separator = ':'
output_array = [['Y', '6', '6'], ['Y:p', 'q:G', 'c:a']]
res = self.evaluate(
string_ops.unsorted_segment_join(
inputs=inputs,
segment_ids=segment_ids,
num_segments=num_segments,
separator=separator))
self.assertAllEqual(res.shape, np.array(output_array).shape)
self.assertAllEqualUnicode(res, output_array)
segment_ids = np.array([1, 0, 1], dtype=np.int64)
num_segments = np.array(2, dtype=np.int64)
res = self.evaluate(
string_ops.unsorted_segment_join(
inputs=inputs,
segment_ids=segment_ids,
num_segments=num_segments,
separator=separator))
self.assertAllEqual(res.shape, np.array(output_array).shape)
self.assertAllEqualUnicode(res, output_array)
def test_basic_tensor(self):
inputs = constant_op.constant([['Y', 'q', 'c'], ['Y', '6', '6'],
['p', 'G', 'a']])
segment_ids = constant_op.constant([1, 0, 1])
num_segments = 2
separator = ':'
output_array = constant_op.constant([['Y', '6', '6'], ['Y:p', 'q:G',
'c:a']])
res = self.evaluate(
string_ops.unsorted_segment_join(
inputs=inputs,
segment_ids=segment_ids,
num_segments=num_segments,
separator=separator))
self.assertAllEqual(res, output_array)
self.assertAllEqual(res.shape, output_array.get_shape())
def test_multiple_segment_join(self):
inputs = [['Y', 'q', 'c'], ['Y', '6', '6'], ['p', 'G', 'a']]
segment_ids_1 = [1, 0, 1]
num_segments_1 = 2
separator_1 = ':'
output_array_1 = [['Y', '6', '6'], ['Y:p', 'q:G', 'c:a']]
res = self.evaluate(
string_ops.unsorted_segment_join(
inputs=inputs,
segment_ids=segment_ids_1,
num_segments=num_segments_1,
separator=separator_1))
self.assertAllEqualUnicode(res, output_array_1)
self.assertAllEqual(res.shape, np.array(output_array_1).shape)
segment_ids_2 = [1, 1]
num_segments_2 = 2
separator_2 = ''
output_array_2 = [['', '', ''], ['YY:p', '6q:G', '6c:a']]
res = self.evaluate(
string_ops.unsorted_segment_join(
inputs=res,
segment_ids=segment_ids_2,
num_segments=num_segments_2,
separator=separator_2))
self.assertAllEqualUnicode(res, output_array_2)
self.assertAllEqual(res.shape, np.array(output_array_2).shape)
@parameterized.parameters([
{
'inputs': [[[['q'], ['s']], [['f'], ['F']], [['h'], ['0']]],
[[['E'], ['j']], [['2'], ['k']], [['N'], ['d']]],
[[['G'], ['M']], [['1'], ['S']], [['N'], ['7']]],
[[['8'], ['W']], [['W'], ['G']], [['j'], ['d']]]],
'segment_ids': [1, 1, 0, 2],
'num_segments':
3,
'separator':
':',
'output_array': [[[['G'], ['M']], [['1'], ['S']], [['N'], ['7']]],
[[['q:E'], ['s:j']], [['f:2'], ['F:k']],
[['h:N'], ['0:d']]],
[[['8'], ['W']], [['W'], ['G']], [['j'], ['d']]]],
},
{
'inputs': [[['Q', 'b'], ['c', 'p']], [['i', '9'], ['n', 'b']],
[['T', 'h'], ['g', 'z']]],
'segment_ids': [[0, 1], [1, 0], [1, 0]],
'num_segments': 2,
'separator': ':',
'output_array': [['Q:n:g', 'b:b:z'], ['c:i:T', 'p:9:h']]
},
{
'inputs': [[['Q', 'b'], ['b', 'p']], [['i', '9'], ['n', 'b']],
[['T', 'h'], ['g', 'z']]],
'segment_ids': [[[2, 1], [0, 0]], [[2, 0], [2, 2]], [[0, 2], [1, 0]]],
'num_segments': 3,
'separator': ':',
'output_array': ['b:p:9:T:z', 'b:g', 'Q:i:n:b:h']
},
{
'inputs': [[['z'], ['h']], [['c'], ['z']], [['V'], ['T']]],
'segment_ids': [0, 1, 1],
'num_segments': 3,
'separator': ':',
'output_array': [[['z'], ['h']], [['c:V'], ['z:T']], [[''], ['']]]
},
])
def test_multiple_cases_with_different_dims(self, inputs, segment_ids,
num_segments, separator,
output_array):
res = self.evaluate(
string_ops.unsorted_segment_join(
inputs=inputs,
segment_ids=segment_ids,
num_segments=num_segments,
separator=separator))
self.assertAllEqualUnicode(res, output_array)
self.assertAllEqual(res.shape, np.array(output_array).shape)
@parameterized.parameters([
{
'separator': '',
'output_array': ['thisisatest']
},
{
'separator': ':',
'output_array': ['this:is:a:test']
},
{
'separator': 'UNK',
'output_array': ['thisUNKisUNKaUNKtest']
},
])
def testSeparator(self, separator, output_array):
inputs = ['this', 'is', 'a', 'test']
segment_ids = [0, 0, 0, 0]
num_segments = 1
res = self.evaluate(
string_ops.unsorted_segment_join(
inputs=inputs,
segment_ids=segment_ids,
num_segments=num_segments,
separator=separator))
self.assertAllEqual(res.shape, np.array(output_array).shape)
self.assertAllEqualUnicode(res, output_array)
def test_fail_segment_id_exceeds_segment_nums(self):
inputs = [['Y', 'q', 'c'], ['Y', '6', '6'], ['p', 'G', 'a']]
segment_ids = [1, 0, 1]
num_segments = 1
separator = ':'
with self.assertRaises(errors_impl.InvalidArgumentError):
self.evaluate(
string_ops.unsorted_segment_join(
inputs=inputs,
segment_ids=segment_ids,
num_segments=num_segments,
separator=separator))
def test_fail_segment_id_dim_does_not_match(self):
inputs = [['Y', 'q', 'c'], ['Y', '6', '6'], ['p', 'G', 'a']]
segment_ids = [1, 0, 1, 1]
num_segments = 2
separator = ':'
if not context.executing_eagerly():
with self.assertRaises(ValueError):
self.evaluate(
string_ops.unsorted_segment_join(
inputs=inputs,
segment_ids=segment_ids,
num_segments=num_segments,
separator=separator))
else:
with self.assertRaises(errors_impl.InvalidArgumentError):
self.evaluate(
string_ops.unsorted_segment_join(
inputs=inputs,
segment_ids=segment_ids,
num_segments=num_segments,
separator=separator))
def test_fail_segment_id_empty_input_non_empty(self):
inputs = [['Y', 'q', 'c'], ['Y', '6', '6'], ['p', 'G', 'a']]
segment_ids = np.array([], dtype=np.int32)
num_segments = 2
separator = ':'
with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)):
self.evaluate(
string_ops.unsorted_segment_join(
inputs=inputs,
segment_ids=segment_ids,
num_segments=num_segments,
separator=separator))
def test_empty_input(self):
inputs = np.array([], dtype=np.bytes_)
segment_ids = [1, 0, 1]
num_segments = 2
separator = ':'
with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)):
self.evaluate(
string_ops.unsorted_segment_join(
inputs=inputs,
segment_ids=segment_ids,
num_segments=num_segments,
separator=separator))
def test_fail_negative_segment_id(self):
inputs = [['Y', 'q', 'c'], ['Y', '6', '6'], ['p', 'G', 'a']]
segment_ids = [-1, 0, -1]
num_segments = 1
separator = ':'
with self.assertRaises(errors_impl.InvalidArgumentError):
self.evaluate(
string_ops.unsorted_segment_join(
inputs=inputs,
segment_ids=segment_ids,
num_segments=num_segments,
separator=separator))
if __name__ == '__main__':
test.main()