wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions 3 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/python/kernel_tests/io_ops/BUILD
+149
View File
@@ -0,0 +1,149 @@
# Tests of TensorFlow IO kernels written using the Python API.
load("//tensorflow:tensorflow.default.bzl", "tf_py_strict_test")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
licenses = ["notice"],
)
tf_py_strict_test(
name = "checkpoint_ops_test",
size = "medium",
srcs = ["checkpoint_ops_test.py"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:checkpoint_ops_gen",
"//tensorflow/python/ops:partitioned_variables",
"//tensorflow/python/ops:variable_scope",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:flags",
"//tensorflow/python/training:saver",
"//third_party/py/numpy",
],
)
tf_py_strict_test(
name = "decode_csv_op_test",
size = "small",
srcs = ["decode_csv_op_test.py"],
deps = [
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/ops:parsing_ops",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
tf_py_strict_test(
name = "io_ops_test",
size = "small",
srcs = ["io_ops_test.py"],
deps = [
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:io_ops",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/util:compat",
],
)
tf_py_strict_test(
name = "parse_single_example_op_test",
size = "small",
srcs = ["parse_single_example_op_test.py"],
deps = [
"//tensorflow/core:protos_all_py",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:sparse_tensor",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:parsing_ops",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:tf_logging",
"//third_party/py/numpy",
],
)
tf_py_strict_test(
name = "parsing_ops_test",
size = "medium",
srcs = ["parsing_ops_test.py"],
deps = [
"//tensorflow/core:protos_all_py",
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:sparse_tensor",
"//tensorflow/python/framework:tensor",
"//tensorflow/python/framework:tensor_shape",
"//tensorflow/python/framework:tensor_util",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:parsing_ops",
"//tensorflow/python/ops/ragged:ragged_concat_ops",
"//tensorflow/python/ops/ragged:ragged_factory_ops",
"//tensorflow/python/ops/ragged:ragged_tensor",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:tf_logging",
"//third_party/py/numpy",
],
)
tf_py_strict_test(
name = "reader_ops_test",
size = "small",
srcs = ["reader_ops_test.py"],
deps = [
"//tensorflow/core:protos_all_py",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/lib/io:tf_record",
"//tensorflow/python/ops:data_flow_ops",
"//tensorflow/python/ops:io_ops",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/util:compat",
],
)
tf_py_strict_test(
name = "record_input_test",
size = "medium",
srcs = ["record_input_test.py"],
deps = [
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/lib/io:tf_record",
"//tensorflow/python/ops:data_flow_ops",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:client_testlib",
],
)
tf_py_strict_test(
name = "save_restore_ops_test",
size = "small",
srcs = ["save_restore_ops_test.py"],
deps = [
"//tensorflow/core:protos_all_py",
"//tensorflow/python/client:session",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:io_ops",
"//tensorflow/python/ops:io_ops_gen",
"//tensorflow/python/platform:client_testlib",
"@absl_py//absl/testing:parameterized",
],
)
tensorflow/python/kernel_tests/io_ops/checkpoint_ops_test.py
+532
View File
@@ -0,0 +1,532 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Functional tests for the ops to generate and execute vocab remapping."""
import os
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_checkpoint_ops
from tensorflow.python.ops import partitioned_variables
from tensorflow.python.ops import variable_scope
from tensorflow.python.ops import variables
from tensorflow.python.platform import flags
from tensorflow.python.platform import test
from tensorflow.python.training import saver
FLAGS = flags.FLAGS
class GenerateVocabRemappingTest(test.TestCase):
"""Tests for the generate_vocab_remapping() method."""
def setUp(self):
self.new_vocab_file = os.path.join(self.get_temp_dir(),
'keyword_shifted.txt')
with open(self.new_vocab_file, 'w') as f:
f.write('\n'.join(['MISSING', 'knitting', 'eminem']) + '\n')
self.old_vocab_file = os.path.join(self.get_temp_dir(),
'keyword.txt')
with open(self.old_vocab_file, 'w') as f:
f.write('\n'.join(['knitting', 'eminem', 'MISSING']) + '\n')
@test_util.run_deprecated_v1
def test_generate_remapping_with_no_vocab_changes(self):
"""Tests where vocab does not change at all."""
remapping, num_present = gen_checkpoint_ops.generate_vocab_remapping(
new_vocab_file=self.old_vocab_file,
old_vocab_file=self.old_vocab_file,
num_new_vocab=3,
new_vocab_offset=0)
expected_remapping = range(0, 3)
expected_num_present = 3
with self.cached_session():
self.assertAllEqual(expected_remapping, self.evaluate(remapping))
self.assertAllEqual(expected_num_present, self.evaluate(num_present))
def test_generate_remapping_with_shifted_vocab(self):
"""Tests where vocab is the same, but shifted / ordered differently."""
remapping, num_present = gen_checkpoint_ops.generate_vocab_remapping(
new_vocab_file=self.new_vocab_file,
old_vocab_file=self.old_vocab_file,
num_new_vocab=3,
new_vocab_offset=0)
expected_remapping = [2, 0, 1]
expected_num_present = 3
with self.cached_session():
self.assertAllEqual(expected_remapping, self.evaluate(remapping))
self.assertAllEqual(expected_num_present, self.evaluate(num_present))
def test_generate_remapping_with_offset(self):
"""Tests offset and num_new_vocab logic."""
remapping, num_present = gen_checkpoint_ops.generate_vocab_remapping(
new_vocab_file=self.new_vocab_file,
old_vocab_file=self.old_vocab_file,
num_new_vocab=1,
new_vocab_offset=1)
expected_remapping = [0]
expected_num_present = 1
with self.cached_session():
self.assertAllEqual(expected_remapping, self.evaluate(remapping))
self.assertAllEqual(expected_num_present, self.evaluate(num_present))
def test_generate_remapping_with_old_vocab_size(self):
"""Tests where old_vocab_size is specified."""
remapping, num_present = gen_checkpoint_ops.generate_vocab_remapping(
new_vocab_file=self.new_vocab_file,
old_vocab_file=self.old_vocab_file,
num_new_vocab=3,
new_vocab_offset=0,
# Old vocabulary becomes ['knitting', 'eminem'].
old_vocab_size=2)
expected_remapping = [-1, 0, 1]
expected_num_present = 2
with self.cached_session():
self.assertAllEqual(expected_remapping, self.evaluate(remapping))
self.assertAllEqual(expected_num_present, self.evaluate(num_present))
class LoadAndRemapMatrixTest(test.TestCase):
"""Tests for the load_and_remap_matrix() op."""
def setUp(self):
ops.reset_default_graph()
self.old_num_rows = 5
self.old_num_cols = 16
self.matrix_value = np.reshape(
range(0, self.old_num_rows * self.old_num_cols), (self.old_num_rows,
self.old_num_cols))
with variable_scope.variable_scope('some_scope'):
matrix = variable_scope.get_variable(
'matrix',
dtype=dtypes.float32,
initializer=constant_op.constant(
self.matrix_value, dtype=dtypes.float32))
self.old_tensor_name = 'some_scope/matrix'
save = saver.Saver([matrix])
with self.cached_session() as sess:
self.evaluate(variables.global_variables_initializer())
self.bundle_file = os.path.join(test.get_temp_dir(), 'bundle_checkpoint')
save.save(sess, self.bundle_file)
def test_load_and_remap_no_missing(self):
"""Tests the op's load and remap where there are no missing entries."""
# No column remapping, new weight matrix has second row, then first row.
row_remapping = [1, 0]
remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=[self.bundle_file],
old_tensor_name=self.old_tensor_name,
row_remapping=row_remapping,
col_remapping=[],
initializing_values=[],
num_rows=2,
num_cols=self.old_num_cols)
with self.cached_session():
self.assertAllClose(self.matrix_value[row_remapping],
self.evaluate(remapped_matrix))
# No row remapping, new weight matrix has third col, then first col.
row_remapping = list(range(self.old_num_rows))
col_remapping = [2, 0]
remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=[self.bundle_file],
old_tensor_name=self.old_tensor_name,
row_remapping=row_remapping,
col_remapping=col_remapping,
initializing_values=[],
num_rows=len(row_remapping),
num_cols=len(col_remapping))
with self.cached_session():
self.assertAllClose(self.matrix_value[row_remapping][:, col_remapping],
self.evaluate(remapped_matrix))
# Both row and column remappings.
row_remapping = [1, 0, 4]
col_remapping = [1, 15]
remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=[self.bundle_file],
old_tensor_name=self.old_tensor_name,
row_remapping=row_remapping,
col_remapping=col_remapping,
initializing_values=[],
num_rows=len(row_remapping),
num_cols=len(col_remapping))
with self.cached_session():
self.assertAllClose(self.matrix_value[row_remapping][:, col_remapping],
self.evaluate(remapped_matrix))
def test_load_and_remap_with_init(self):
"""Tests the op's load and remap where there are missing entries."""
init_val = 42
remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=[self.bundle_file],
old_tensor_name=self.old_tensor_name,
row_remapping=[2, -1, 0],
col_remapping=[1, -1],
initializing_values=[init_val] * 4,
num_rows=3,
num_cols=2)
expected_remapped_matrix = np.reshape(
[33, init_val, init_val, init_val, 1, init_val], [3, 2])
with self.cached_session():
self.assertAllClose(expected_remapped_matrix,
self.evaluate(remapped_matrix))
def test_load_and_remap_all_missing_rows(self):
"""Tests when all the rows are missing and need to be initialized."""
num_rows = 7
initializing_values = [42] * num_rows * self.old_num_cols
remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=[self.bundle_file],
old_tensor_name=self.old_tensor_name,
row_remapping=[-1] * num_rows,
col_remapping=[],
initializing_values=initializing_values,
num_rows=num_rows,
num_cols=self.old_num_cols)
with self.cached_session():
self.assertAllClose(
np.reshape(initializing_values, (num_rows, self.old_num_cols)),
self.evaluate(remapped_matrix))
def test_load_and_remap_all_missing_rows_and_cols(self):
"""Tests when all the rows & cols are missing and need to be initialized."""
num_rows = 7
num_cols = 4
initializing_values = [42] * num_rows * num_cols
remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=[self.bundle_file],
old_tensor_name=self.old_tensor_name,
row_remapping=[-1] * num_rows,
col_remapping=[-1] * num_cols,
initializing_values=initializing_values,
num_rows=num_rows,
num_cols=num_cols)
with self.cached_session():
self.assertAllClose(
np.reshape(initializing_values, (num_rows, num_cols)),
self.evaluate(remapped_matrix))
def test_load_and_remap_invalid_dims(self):
ckpt_path = constant_op.constant(
'/tmp/warm_starting_util_test5kl2a3pc/tmpph76tep2/model-0',
shape=[],
dtype=dtypes.string)
old_tensor_name = constant_op.constant(
'/tmp/warm_starting_util_test5kl2a3pc/tmpph76tep2/model-0',
shape=[],
dtype=dtypes.string)
row_remapping = constant_op.constant(0, shape=[], dtype=dtypes.int64)
col_remapping = constant_op.constant(3, shape=[3], dtype=dtypes.int64)
initializing_values = constant_op.constant([],
shape=[0, 1],
dtype=dtypes.float32)
with self.cached_session(), self.assertRaisesRegex(
(ValueError, errors.InvalidArgumentError), 'tensor must be 1-D'):
self.evaluate(
gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=ckpt_path,
old_tensor_name=old_tensor_name,
row_remapping=row_remapping,
col_remapping=col_remapping,
initializing_values=initializing_values,
num_rows=1,
num_cols=1))
@test_util.run_deprecated_v1
def test_load_and_remap_empty_old_tensor_name(self):
old_tensor_name_placeholder = array_ops.placeholder(dtypes.string)
remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=[self.bundle_file],
old_tensor_name=old_tensor_name_placeholder,
row_remapping=[0],
col_remapping=[],
initializing_values=[],
num_rows=1,
num_cols=self.old_num_cols,
)
with self.cached_session() as sess:
with self.assertRaisesRegex(
errors.InvalidArgumentError,
'old_tensor_name.*must have exactly one element',
):
sess.run(remapped_matrix, feed_dict={old_tensor_name_placeholder: []})
def test_load_and_remap_out_of_bounds_row_remapping(self):
# self.old_num_rows is 5. We use 5 (out of bounds) in row_remapping.
with self.cached_session(), self.assertRaisesRegex(
errors.InvalidArgumentError, 'Row remapping index 5 is out of bounds'
):
self.evaluate(
gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=[self.bundle_file],
old_tensor_name=self.old_tensor_name,
row_remapping=[0, 5],
col_remapping=[],
initializing_values=[],
num_rows=2,
num_cols=self.old_num_cols,
)
)
def test_load_and_remap_out_of_bounds_col_remapping(self):
# self.old_num_cols is 16. We use 16 (out of bounds) in col_remapping.
with self.cached_session(), self.assertRaisesRegex(
errors.InvalidArgumentError, 'Col remapping index 16 is out of bounds'
):
self.evaluate(
gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=[self.bundle_file],
old_tensor_name=self.old_tensor_name,
row_remapping=list(range(self.old_num_rows)),
col_remapping=[0, 16],
initializing_values=[],
num_rows=self.old_num_rows,
num_cols=2,
)
)
@test_util.run_deprecated_v1
def test_load_and_remap_invalid_remapping(self):
"""Tests that errors are raised when an ID maps to multiple new IDs.
(This should usually not happen when using public APIs).
"""
invalid_remapping = [1, 0, 0, 0, 1, 2]
# Invalid row remapping.
remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=[self.bundle_file],
old_tensor_name=self.old_tensor_name,
row_remapping=invalid_remapping,
col_remapping=[],
initializing_values=[],
num_rows=len(invalid_remapping),
num_cols=self.old_num_cols)
with self.cached_session(), self.assertRaises(errors.UnimplementedError):
self.evaluate(remapped_matrix)
# Invalid column remapping.
remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=[self.bundle_file],
old_tensor_name=self.old_tensor_name,
row_remapping=list(range(self.old_num_rows)),
col_remapping=invalid_remapping,
initializing_values=[],
num_rows=self.old_num_rows,
num_cols=len(invalid_remapping))
with self.cached_session(), self.assertRaises(errors.UnimplementedError):
self.evaluate(remapped_matrix)
@test_util.run_deprecated_v1
def test_load_and_remap_incorrect_initializing_values(self):
"""Tests that errors are raised with incorrect number of init values."""
remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=[self.bundle_file],
old_tensor_name=self.old_tensor_name,
row_remapping=[2, -1, 0],
col_remapping=[1, -1],
# Too few initializing values - there should be 4. For some reason,
# initializing_values must contain no element (instead of 3 or fewer) to
# ensure that a seg fault would reliably occur if the check raising the
# InvalidArgumentError were not present.
initializing_values=[],
num_rows=3,
num_cols=2)
with self.cached_session(), self.assertRaises(errors.InvalidArgumentError):
self.evaluate(remapped_matrix)
remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=[self.bundle_file],
old_tensor_name=self.old_tensor_name,
row_remapping=[2, -1, 0],
col_remapping=[1, -1],
# Too many initializing values - there should be 4.
initializing_values=[0] * 5,
num_rows=3,
num_cols=2)
with self.cached_session(), self.assertRaises(errors.InvalidArgumentError):
self.evaluate(remapped_matrix)
class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase):
"""Tests for the load_and_remap_matrix() op.
(Specifically focused on the max_rows_in_memory arg and its effects on
TensorBundle's BundleReader and TensorSlice logic).
"""
def _test_loading_variable_with_max_rows(self, np_value, partitioner,
max_rows_in_memory):
"""Helper function for various tests using max_rows_in_memory."""
ops.reset_default_graph()
old_tensor_name = 'matrix_to_load_and_remap'
matrix = variable_scope.get_variable(
old_tensor_name,
dtype=dtypes.float32,
initializer=constant_op.constant(np_value, dtype=dtypes.float32),
partitioner=partitioner)
with self.cached_session() as sess:
ckpt_path = os.path.join(test.get_temp_dir(), 'temp_ckpt')
save = saver.Saver([matrix])
self.evaluate(variables.global_variables_initializer())
save.save(sess, ckpt_path)
num_rows, num_cols = np_value.shape
# Tests loading the entire tensor (except reversed).
remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=ckpt_path,
old_tensor_name=old_tensor_name,
# Simply reverses the rows of the matrix.
row_remapping=list(range(num_rows - 1, -1, -1)),
col_remapping=[],
initializing_values=[],
num_rows=num_rows,
num_cols=num_cols,
max_rows_in_memory=max_rows_in_memory)
self.assertAllClose(np_value[::-1], self.evaluate(remapped_matrix))
# Tests loading the tensor (except for the first and last rows), with
# uninitialized values. Requires num_rows to be at least 3 since we're
# skipping the first and last rows.
self.assertGreater(num_rows, 2)
prefix_rows = 2
suffix_rows = 3
remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=ckpt_path,
old_tensor_name=old_tensor_name,
# Reverses the rows of the matrix, then prepends and appends
# uninitialized rows.
row_remapping=([-1] * prefix_rows + list(range(1, num_rows - 1)) +
[-1] * suffix_rows),
col_remapping=[],
initializing_values=[42] * (prefix_rows + suffix_rows) * num_cols,
num_rows=num_rows - 2 + prefix_rows + suffix_rows,
num_cols=num_cols,
max_rows_in_memory=max_rows_in_memory)
self.assertAllClose(
np.vstack([
np.tile(42, [prefix_rows, num_cols]), np_value[1:-1],
np.tile(42, [suffix_rows, num_cols])
]), self.evaluate(remapped_matrix))
# Tests when everything is taken from initializing_values.
new_rows = 7
initializing_values = [42] * new_rows * num_cols
remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
ckpt_path=ckpt_path,
old_tensor_name=old_tensor_name,
# Nothing is loaded from the old tensor.
row_remapping=[-1] * new_rows,
col_remapping=[],
initializing_values=initializing_values,
num_rows=new_rows,
num_cols=num_cols,
max_rows_in_memory=max_rows_in_memory)
self.assertAllClose(
np.reshape(initializing_values, (new_rows, num_cols)),
self.evaluate(remapped_matrix))
@test_util.run_deprecated_v1
def test_loading_rows_divisible_by_max_rows(self):
"""Tests loading normal var when rows are evenly divisible by max_rows."""
self._test_loading_variable_with_max_rows(
np_value=np.reshape(list(range(0, 36)), (9, 4)),
partitioner=None,
# 9 is evenly divisible by 3.
max_rows_in_memory=3)
@test_util.run_deprecated_v1
def test_loading_rows_not_divisible_by_max_rows(self):
"""Tests loading normal var when rows aren't divisible by max_rows."""
self._test_loading_variable_with_max_rows(
np_value=np.reshape(list(range(0, 36)), (9, 4)),
partitioner=None,
# 9 is not evenly divisible by 4.
max_rows_in_memory=4)
@test_util.run_deprecated_v1
def test_loading_rows_less_than_max_rows(self):
"""Tests loading normal var as a single slice.
(When the specified max_rows_in_memory is larger than the number of rows)
"""
self._test_loading_variable_with_max_rows(
np_value=np.reshape(list(range(0, 36)), (9, 4)),
partitioner=None,
# 10 > 9.
max_rows_in_memory=10)
@test_util.run_deprecated_v1
def test_loading_no_max_rows(self):
"""Tests loading normal var as a single slice with no valid max_rows."""
self._test_loading_variable_with_max_rows(
np_value=np.reshape(list(range(0, 18)), (6, 3)),
partitioner=None,
max_rows_in_memory=-1)
@test_util.run_deprecated_v1
def test_loading_partitions_equals_max_rows(self):
"""Tests loading partitioned var sliced on partition boundary."""
self._test_loading_variable_with_max_rows(
np_value=np.reshape(list(range(0, 36)), (9, 4)),
partitioner=partitioned_variables.fixed_size_partitioner(3),
# With a tensor of shape [9, 3] and 3 partitions, each partition has
# exactly 3 rows.
max_rows_in_memory=3)
@test_util.run_deprecated_v1
def test_loading_partitions_greater_than_max_rows(self):
"""Tests loading partitioned var with more slices than partitions."""
self._test_loading_variable_with_max_rows(
np_value=np.reshape(list(range(0, 36)), (9, 4)),
partitioner=partitioned_variables.fixed_size_partitioner(3),
# Even though each partition has 3 rows, we'll only load the tensor one
# row at a time.
max_rows_in_memory=1)
@test_util.run_deprecated_v1
def test_loading_partitions_less_than_max_rows(self):
"""Tests loading partitioned var as a single slice.
(When the specified max_rows_in_memory is larger than the number of rows)
"""
self._test_loading_variable_with_max_rows(
np_value=np.reshape(list(range(0, 36)), (9, 4)),
partitioner=partitioned_variables.fixed_size_partitioner(3),
max_rows_in_memory=10)
@test_util.run_deprecated_v1
def test_loading_partitions_no_max_rows(self):
"""Tests loading partitioned var as single slice with no valid max_rows."""
self._test_loading_variable_with_max_rows(
np_value=np.reshape(list(range(0, 36)), (9, 4)),
partitioner=partitioned_variables.fixed_size_partitioner(3),
max_rows_in_memory=-1)
if __name__ == '__main__':
test.main()
tensorflow/python/kernel_tests/io_ops/decode_csv_op_test.py
+330
View File
@@ -0,0 +1,330 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for DecodeCSV op from parsing_ops."""
import numpy as np
from tensorflow.python.eager import context
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import errors
from tensorflow.python.ops import parsing_ops
from tensorflow.python.platform import test
class DecodeCSVOpTest(test.TestCase):
def _test(self, args, expected_out=None, expected_err_re=None):
if expected_err_re is None:
decode = parsing_ops.decode_csv(**args)
out = self.evaluate(decode)
for i, field in enumerate(out):
if field.dtype == np.float32 or field.dtype == np.float64:
self.assertAllClose(field, expected_out[i])
else:
self.assertAllEqual(field, expected_out[i])
else:
with self.assertRaisesWithPredicateMatch(Exception, expected_err_re):
decode = parsing_ops.decode_csv(**args)
self.evaluate(decode)
def testSimple(self):
args = {
"records": ["1", "2", '"3"'],
"record_defaults": [[1]],
}
expected_out = [[1, 2, 3]]
self._test(args, expected_out)
def testSimpleWithScalarDefaults(self):
args = {
"records": ["1,4", "2,5", "3,6"],
"record_defaults": [1, 2],
}
expected_out = [[1, 2, 3], [4, 5, 6]]
self._test(args, expected_out)
def testSimpleWith2DDefaults(self):
args = {
"records": ["1", "2", "3"],
"record_defaults": [[[0]]],
}
if context.executing_eagerly():
err_spec = errors.InvalidArgumentError, (
"Each record default should be at "
"most rank 1")
else:
err_spec = ValueError, "Shape must be at most rank 1 but is rank 2"
with self.assertRaisesWithPredicateMatch(*err_spec):
self._test(args)
def testSimpleNoQuoteDelimiter(self):
args = {
"records": ["1", "2", '"3"'],
"record_defaults": [[""]],
"use_quote_delim": False,
}
expected_out = [[b"1", b"2", b'"3"']]
self._test(args, expected_out)
def testScalar(self):
args = {"records": '1,""', "record_defaults": [[3], [4]]}
expected_out = [1, 4]
self._test(args, expected_out)
def test2D(self):
args = {"records": [["1", "2"], ['""', "4"]], "record_defaults": [[5]]}
expected_out = [[[1, 2], [5, 4]]]
self._test(args, expected_out)
def test2DNoQuoteDelimiter(self):
args = {
"records": [["1", "2"], ['""', '"']],
"record_defaults": [[""]],
"use_quote_delim": False
}
expected_out = [[[b"1", b"2"], [b'""', b'"']]]
self._test(args, expected_out)
def testDouble(self):
args = {
"records": ["1.0", "-1.79e+308", '"1.79e+308"'],
"record_defaults": [np.array([], dtype=np.double)],
}
expected_out = [[1.0, -1.79e+308, 1.79e+308]]
self._test(args, expected_out)
def testInt64(self):
args = {
"records": ["1", "2", '"2147483648"'],
"record_defaults": [np.array([], dtype=np.int64)],
}
expected_out = [[1, 2, 2147483648]]
self._test(args, expected_out)
def testComplexString(self):
args = {
"records": ['"1.0"', '"ab , c"', '"a\nbc"', '"ab""c"', " abc "],
"record_defaults": [["1"]]
}
expected_out = [[b"1.0", b"ab , c", b"a\nbc", b'ab"c', b" abc "]]
self._test(args, expected_out)
def testMultiRecords(self):
args = {
"records": ["1.0,4,aa", "0.2,5,bb", "3,6,cc"],
"record_defaults": [[1.0], [1], ["aa"]]
}
expected_out = [[1.0, 0.2, 3], [4, 5, 6], [b"aa", b"bb", b"cc"]]
self._test(args, expected_out)
def testNA(self):
args = {
"records": ["2.0,NA,aa", "NA,5,bb", "3,6,NA"],
"record_defaults": [[0.0], [0], [""]],
"na_value": "NA"
}
expected_out = [[2.0, 0.0, 3], [0, 5, 6], [b"aa", b"bb", b""]]
self._test(args, expected_out)
def testWithDefaults(self):
args = {
"records": [",1,", "0.2,3,bcd", "3.0,,"],
"record_defaults": [[1.0], [0], ["a"]]
}
expected_out = [[1.0, 0.2, 3.0], [1, 3, 0], [b"a", b"bcd", b"a"]]
self._test(args, expected_out)
def testWithDefaultsAndNoQuoteDelimiter(self):
args = {
"records": [",1,", "0.2,3,bcd", '3.0,,"'],
"record_defaults": [[1.0], [0], ["a"]],
"use_quote_delim": False,
}
expected_out = [[1.0, 0.2, 3.0], [1, 3, 0], [b"a", b"bcd", b"\""]]
self._test(args, expected_out)
def testWithTabDelim(self):
args = {
"records": ["1\t1", "0.2\t3", "3.0\t"],
"record_defaults": [[1.0], [0]],
"field_delim": "\t"
}
expected_out = [[1.0, 0.2, 3.0], [1, 3, 0]]
self._test(args, expected_out)
def testWithoutDefaultsError(self):
args = {
"records": [",1", "0.2,3", "3.0,"],
"record_defaults": [[1.0], np.array([], dtype=np.int32)]
}
self._test(
args, expected_err_re="Field 1 is required but missing in record 2!")
def testWrongFieldIntError(self):
args = {
"records": [",1", "0.2,234a", "3.0,2"],
"record_defaults": [[1.0], np.array([], dtype=np.int32)]
}
self._test(
args, expected_err_re="Field 1 in record 1 is not a valid int32: 234a")
def testOutOfRangeError(self):
args = {
"records": ["1", "9999999999999999999999999", "3"],
"record_defaults": [[1]]
}
self._test(
args, expected_err_re="Field 0 in record 1 is not a valid int32: ")
def testWrongFieldFloatError(self):
args = {
"records": [",1", "0.2,2", "3.0adf,3"],
"record_defaults": [[1.0], np.array([], dtype=np.int32)]
}
self._test(
args, expected_err_re="Field 0 in record 2 is not a valid float: ")
def testWrongFieldStringError(self):
args = {"records": ['"1,a,"', "0.22", 'a"bc'], "record_defaults": [["a"]]}
self._test(
args, expected_err_re="Unquoted fields cannot have quotes/CRLFs inside")
def testWrongDefaults(self):
args = {"records": [",1", "0.2,2", "3.0adf,3"], "record_defaults": [[1.0]]}
self._test(args, expected_err_re="Expect 1 fields but have 2 in record 0")
def testShortQuotedString(self):
args = {
"records": ["\""],
"record_defaults": [["default"]],
}
self._test(
args, expected_err_re="Quoted field has to end with quote followed.*")
def testSelectCols(self):
args = {
"records": [",,", "4,5,6"],
"record_defaults": [[1], [2]],
"select_cols": [0, 1]
}
expected_out = [[1, 4], [2, 5]]
self._test(args, expected_out)
def testSelectColsInclLast(self):
# The last col is a edge-casey; add test for that
args = {
"records": [",,", "4,5,6"],
"record_defaults": [[0], [1], [2]],
"select_cols": [0, 1, 2]
}
expected_out = [[0, 4], [1, 5], [2, 6]]
self._test(args, expected_out)
def testWrongSelectColsInclLast(self):
# The last col is a edge-casey; add test for that
args = {
"records": [",,", "4,5,6"],
"record_defaults": [[0], [1], [2]],
"select_cols": [0, 1, 3]
}
self._test(args, expected_err_re="Expect 3 fields but have 2 in record 0")
def testWrongSelectColsLen(self):
args = {
"records": ["1,2,3", "4,5,6"],
"record_defaults": [[0], [0], [0]],
"select_cols": [0]
}
with self.assertRaisesWithPredicateMatch(
ValueError, "Length of select_cols and record_defaults do not match."):
self._test(args)
def testWrongSelectColsSorting(self):
args = {
"records": ["1,2,3"],
"record_defaults": [[0], [1]],
"select_cols": [1, 0]
}
with self.assertRaisesWithPredicateMatch(
ValueError, "select_cols is not strictly increasing."):
self._test(args)
def testWrongSelectColsIndicesNegative(self):
args = {
"records": ["1,2,3"],
"record_defaults": [[0], [1]],
"select_cols": [-1, 0] # -1 is not a valid index
}
with self.assertRaisesWithPredicateMatch(
ValueError, "select_cols contains negative values."):
self._test(args)
def testWrongSelectColsIndicesTooHigh(self):
args = {
"records": ["1,2,3"],
"record_defaults": [[0], [1]],
"select_cols": [0, 3] # 3 is not a valid index
}
# Only successfully parses one of the columns
self._test(args, expected_err_re="Expect 2 fields but have 1 in record 0")
def testNumpyAttribute(self):
args = {
"record_defaults": np.zeros(5),
"records": constant_op.constant("1,2,3,4,5"),
}
if context.executing_eagerly():
self._test(args, expected_out=[1, 2, 3, 4, 5])
else:
self._test(args, expected_err_re="Expected list for 'record_defaults'")
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/io_ops/io_ops_test.py
+139
View File
@@ -0,0 +1,139 @@
# -*- coding: utf-8 -*-
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tensorflow.python.ops.io_ops."""
import os
import shutil
import tempfile
from tensorflow.python.framework import test_util
from tensorflow.python.ops import io_ops
from tensorflow.python.platform import test
from tensorflow.python.util import compat
class IoOpsTest(test.TestCase):
@test_util.run_deprecated_v1
def testReadFile(self):
cases = ['', 'Some contents', 'Неки садржаји на српском']
for contents in cases:
contents = compat.as_bytes(contents)
with tempfile.NamedTemporaryFile(
prefix='ReadFileTest', dir=self.get_temp_dir(), delete=False) as temp:
temp.write(contents)
with self.cached_session():
read = io_ops.read_file(temp.name)
self.assertEqual([], read.get_shape())
self.assertEqual(self.evaluate(read), contents)
os.remove(temp.name)
def testWriteFile(self):
cases = ['', 'Some contents']
for contents in cases:
contents = compat.as_bytes(contents)
with tempfile.NamedTemporaryFile(
prefix='WriteFileTest', dir=self.get_temp_dir(),
delete=False) as temp:
pass
with self.cached_session() as sess:
w = io_ops.write_file(temp.name, contents)
self.evaluate(w)
with open(temp.name, 'rb') as f:
file_contents = f.read()
self.assertEqual(file_contents, contents)
os.remove(temp.name)
def testWriteFileCreateDir(self):
cases = ['', 'Some contents']
for contents in cases:
contents = compat.as_bytes(contents)
subdir = os.path.join(self.get_temp_dir(), 'subdir1')
filepath = os.path.join(subdir, 'subdir2', 'filename')
with self.cached_session() as sess:
w = io_ops.write_file(filepath, contents)
self.evaluate(w)
with open(filepath, 'rb') as f:
file_contents = f.read()
self.assertEqual(file_contents, contents)
shutil.rmtree(subdir)
def _subset(self, files, indices):
return set(
compat.as_bytes(files[i].name) for i in range(len(files))
if i in indices)
@test_util.run_deprecated_v1
def testMatchingFiles(self):
cases = [
'ABcDEF.GH', 'ABzDEF.GH', 'ABasdfjklDEF.GH', 'AB3DEF.GH', 'AB4DEF.GH',
'ABDEF.GH', 'XYZ'
]
files = [
tempfile.NamedTemporaryFile(
prefix=c, dir=self.get_temp_dir(), delete=True) for c in cases
]
with self.cached_session():
# Test exact match without wildcards.
for f in files:
self.assertEqual(
io_ops.matching_files(f.name).eval(), compat.as_bytes(f.name))
# We will look for files matching "ABxDEF.GH*" where "x" is some wildcard.
directory_path = files[0].name[:files[0].name.find(cases[0])]
pattern = directory_path + 'AB%sDEF.GH*'
self.assertEqual(
set(io_ops.matching_files(pattern % 'z').eval()),
self._subset(files, [1]))
self.assertEqual(
set(io_ops.matching_files(pattern % '?').eval()),
self._subset(files, [0, 1, 3, 4]))
self.assertEqual(
set(io_ops.matching_files(pattern % '*').eval()),
self._subset(files, [0, 1, 2, 3, 4, 5]))
# NOTE(mrry): Windows uses PathMatchSpec to match file patterns, which
# does not support the following expressions.
if os.name != 'nt':
self.assertEqual(
set(io_ops.matching_files(pattern % '[cxz]').eval()),
self._subset(files, [0, 1]))
self.assertEqual(
set(io_ops.matching_files(pattern % '[0-9]').eval()),
self._subset(files, [3, 4]))
# Test an empty list input.
self.assertItemsEqual(io_ops.matching_files([]).eval(), [])
# Test multiple exact filenames.
self.assertItemsEqual(
io_ops.matching_files([
files[0].name, files[1].name, files[2].name]).eval(),
self._subset(files, [0, 1, 2]))
# Test multiple globs.
self.assertItemsEqual(
io_ops.matching_files([
pattern % '?', directory_path + 'X?Z*']).eval(),
self._subset(files, [0, 1, 3, 4, 6]))
for f in files:
f.close()
if __name__ == '__main__':
test.main()
tensorflow/python/kernel_tests/io_ops/parse_single_example_op_test.py
+994
View File
@@ -0,0 +1,994 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tensorflow.ops.parsing_ops."""
import itertools
import numpy as np
from tensorflow.core.example import example_pb2
from tensorflow.core.example import feature_pb2
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import ops
from tensorflow.python.framework import sparse_tensor
from tensorflow.python.framework import tensor_shape
from tensorflow.python.framework import test_util
from tensorflow.python.ops import parsing_ops
from tensorflow.python.platform import test
from tensorflow.python.platform import tf_logging
# Helpers for creating Example objects
example = example_pb2.Example
feature = feature_pb2.Feature
features = lambda d: feature_pb2.Features(feature=d)
bytes_feature = lambda v: feature(bytes_list=feature_pb2.BytesList(value=v))
int64_feature = lambda v: feature(int64_list=feature_pb2.Int64List(value=v))
float_feature = lambda v: feature(float_list=feature_pb2.FloatList(value=v))
# Helpers for creating SequenceExample objects
feature_list = lambda l: feature_pb2.FeatureList(feature=l)
feature_lists = lambda d: feature_pb2.FeatureLists(feature_list=d)
sequence_example = example_pb2.SequenceExample
def empty_sparse(dtype, shape=None):
if shape is None:
shape = [0]
return (np.empty(shape=(0, len(shape)), dtype=np.int64),
np.array([], dtype=dtype), np.array(shape, dtype=np.int64))
def flatten(list_of_lists):
"""Flatten one level of nesting."""
return itertools.chain.from_iterable(list_of_lists)
def flatten_values_tensors_or_sparse(tensors_list):
"""Flatten each SparseTensor object into 3 Tensors for session.run()."""
return list(
flatten([[v.indices, v.values, v.dense_shape] if isinstance(
v, sparse_tensor.SparseTensor) else [v] for v in tensors_list]))
def _compare_output_to_expected(tester, dict_tensors, expected_tensors,
flat_output):
tester.assertEqual(set(dict_tensors.keys()), set(expected_tensors.keys()))
i = 0 # Index into the flattened output of session.run()
for k, v in dict_tensors.items():
expected_v = expected_tensors[k]
tf_logging.info("Comparing key: %s", k)
if isinstance(v, sparse_tensor.SparseTensor):
# Three outputs for SparseTensor : indices, values, shape.
tester.assertEqual([k, len(expected_v)], [k, 3])
tester.assertAllEqual(expected_v[0], flat_output[i])
tester.assertAllEqual(expected_v[1], flat_output[i + 1])
tester.assertAllEqual(expected_v[2], flat_output[i + 2])
i += 3
else:
# One output for standard Tensor.
tester.assertAllEqual(expected_v, flat_output[i])
i += 1
class ParseExampleTest(test.TestCase):
def _test(self, kwargs, expected_values=None, expected_err=None):
with self.cached_session() as sess:
if expected_err:
with self.assertRaisesWithPredicateMatch(expected_err[0],
expected_err[1]):
out = parsing_ops.parse_single_example(**kwargs)
sess.run(flatten_values_tensors_or_sparse(out.values()))
return
else:
# Returns dict w/ Tensors and SparseTensors.
out = parsing_ops.parse_single_example(**kwargs)
# Also include a test with the example names specified to retain
# code coverage of the unfused version, and ensure that the two
# versions produce the same results.
out_with_example_name = parsing_ops.parse_single_example(
example_names="name", **kwargs)
for result_dict in [out, out_with_example_name]:
result = flatten_values_tensors_or_sparse(result_dict.values())
# Check values.
tf_result = self.evaluate(result)
_compare_output_to_expected(self, result_dict, expected_values,
tf_result)
for k, f in kwargs["features"].items():
if isinstance(f, parsing_ops.FixedLenFeature) and f.shape is not None:
self.assertEqual(tuple(out[k].get_shape().as_list()), f.shape)
elif isinstance(f, parsing_ops.VarLenFeature):
self.assertEqual(
tuple(out[k].indices.get_shape().as_list()), (None, 1))
self.assertEqual(tuple(out[k].values.get_shape().as_list()), (None,))
self.assertEqual(
tuple(out[k].dense_shape.get_shape().as_list()), (1,))
@test_util.run_deprecated_v1
def testEmptySerializedWithAllDefaults(self):
sparse_name = "st_a"
a_name = "a"
b_name = "b"
c_name = "c:has_a_tricky_name"
a_default = [0, 42, 0]
b_default = np.random.rand(3, 3).astype(bytes)
c_default = np.random.rand(2).astype(np.float32)
expected_st_a = ( # indices, values, shape
np.empty((0, 1), dtype=np.int64), # indices
np.empty((0,), dtype=np.int64), # sp_a is DT_INT64
np.array([0], dtype=np.int64)) # max_elems = 0
expected_output = {
sparse_name: expected_st_a,
a_name: np.array([a_default]),
b_name: np.array(b_default),
c_name: np.array(c_default),
}
self._test({
"serialized": ops.convert_to_tensor(""),
"features": {
sparse_name:
parsing_ops.VarLenFeature(dtypes.int64),
a_name:
parsing_ops.FixedLenFeature(
(1, 3), dtypes.int64, default_value=a_default),
b_name:
parsing_ops.FixedLenFeature(
(3, 3), dtypes.string, default_value=b_default),
c_name:
parsing_ops.FixedLenFeature(
(2,), dtypes.float32, default_value=c_default),
}
}, expected_output)
def testEmptySerializedWithoutDefaultsShouldFail(self):
input_features = {
"st_a":
parsing_ops.VarLenFeature(dtypes.int64),
"a":
parsing_ops.FixedLenFeature(
(1, 3), dtypes.int64, default_value=[0, 42, 0]),
"b":
parsing_ops.FixedLenFeature(
(3, 3),
dtypes.string,
default_value=np.random.rand(3, 3).astype(bytes)),
# Feature "c" is missing a default, this gap will cause failure.
"c":
parsing_ops.FixedLenFeature(
(2,), dtype=dtypes.float32),
}
# Edge case where the key is there but the feature value is empty
original = example(features=features({"c": feature()}))
self._test(
{
"serialized": original.SerializeToString(),
"features": input_features,
},
expected_err=(errors_impl.OpError,
"Feature: c \\(data type: float\\) is required"))
# Standard case of missing key and value.
self._test(
{
"serialized": "",
"features": input_features,
},
expected_err=(errors_impl.OpError,
"Feature: c \\(data type: float\\) is required"))
def testDenseNotMatchingShapeShouldFail(self):
original = example(features=features({
"a": float_feature([-1, -1]),
}))
serialized = original.SerializeToString()
self._test(
{
"serialized": ops.convert_to_tensor(serialized),
"features": {
"a": parsing_ops.FixedLenFeature((1, 3), dtypes.float32)
}
},
# TODO(mrry): Consider matching the `io.parse_example()` error message.
expected_err=(errors_impl.OpError, "Key: a."))
def testDenseDefaultNoShapeShouldFail(self):
original = example(features=features({
"a": float_feature([1, 1, 3]),
}))
serialized = original.SerializeToString()
self._test(
{
"serialized": ops.convert_to_tensor(serialized),
"features": {
"a": parsing_ops.FixedLenFeature(None, dtypes.float32)
}
},
expected_err=(ValueError, "Missing shape for feature a"))
@test_util.run_deprecated_v1
def testSerializedContainingSparse(self):
original = [
example(features=features({
"st_c": float_feature([3, 4])
})),
example(features=features({
"st_c": float_feature([]), # empty float list
})),
example(features=features({
"st_d": feature(), # feature with nothing in it
})),
example(features=features({
"st_c": float_feature([1, 2, -1]),
"st_d": bytes_feature([b"hi"])
}))
]
expected_outputs = [{
"st_c": (np.array([[0], [1]], dtype=np.int64),
np.array([3.0, 4.0], dtype=np.float32),
np.array([2], dtype=np.int64)),
"st_d":
empty_sparse(bytes)
}, {
"st_c": empty_sparse(np.float32),
"st_d": empty_sparse(bytes)
}, {
"st_c": empty_sparse(np.float32),
"st_d": empty_sparse(bytes)
}, {
"st_c": (np.array([[0], [1], [2]], dtype=np.int64),
np.array([1.0, 2.0, -1.0], dtype=np.float32),
np.array([3], dtype=np.int64)),
"st_d": (np.array([[0]], dtype=np.int64), np.array(["hi"], dtype=bytes),
np.array([1], dtype=np.int64))
}]
for proto, expected_output in zip(original, expected_outputs):
self._test({
"serialized": ops.convert_to_tensor(proto.SerializeToString()),
"features": {
"st_c": parsing_ops.VarLenFeature(dtypes.float32),
"st_d": parsing_ops.VarLenFeature(dtypes.string)
},
}, expected_output)
def testSerializedContainingSparseFeature(self):
original = [
example(features=features({
"val": float_feature([3, 4]),
"idx": int64_feature([5, 10])
})),
example(features=features({
"val": float_feature([]), # empty float list
"idx": int64_feature([])
})),
example(features=features({
"val": feature(), # feature with nothing in it
# missing idx feature
})),
example(features=features({
"val": float_feature([1, 2, -1]),
"idx":
int64_feature([0, 9, 3]) # unsorted
}))
]
expected_outputs = [{
"sp": (np.array([[5], [10]], dtype=np.int64),
np.array([3.0, 4.0], dtype=np.float32),
np.array([13], dtype=np.int64))
}, {
"sp": empty_sparse(np.float32, shape=[13])
}, {
"sp": empty_sparse(np.float32, shape=[13])
}, {
"sp": (np.array([[0], [3], [9]], dtype=np.int64),
np.array([1.0, -1.0, 2.0], dtype=np.float32),
np.array([13], dtype=np.int64))
}]
for proto, expected_output in zip(original, expected_outputs):
self._test({
"serialized": ops.convert_to_tensor(proto.SerializeToString()),
"features": {
"sp":
parsing_ops.SparseFeature(["idx"], "val", dtypes.float32,
[13])
}
}, expected_output)
def testSerializedContainingSparseFeatureReuse(self):
original = [
example(features=features({
"val1": float_feature([3, 4]),
"val2": float_feature([5, 6]),
"idx": int64_feature([5, 10])
})),
example(features=features({
"val1": float_feature([]), # empty float list
"idx": int64_feature([])
})),
]
expected_outputs = [{
"sp1": (np.array([[5], [10]], dtype=np.int64),
np.array([3.0, 4.0], dtype=np.float32),
np.array([13], dtype=np.int64)),
"sp2": (np.array([[5], [10]], dtype=np.int64),
np.array([5.0, 6.0], dtype=np.float32),
np.array([7], dtype=np.int64))
}, {
"sp1": empty_sparse(np.float32, shape=[13]),
"sp2": empty_sparse(np.float32, shape=[7])
}]
for proto, expected_output in zip(original, expected_outputs):
self._test({
"serialized": ops.convert_to_tensor(proto.SerializeToString()),
"features": {
"sp1":
parsing_ops.SparseFeature("idx", "val1", dtypes.float32, 13),
"sp2":
parsing_ops.SparseFeature(
"idx",
"val2",
dtypes.float32,
size=7,
already_sorted=True)
}
}, expected_output)
def testSerializedContaining3DSparseFeature(self):
original = [
example(features=features({
"val": float_feature([3, 4]),
"idx0": int64_feature([5, 10]),
"idx1": int64_feature([0, 2]),
})),
example(features=features({
"val": float_feature([]), # empty float list
"idx0": int64_feature([]),
"idx1": int64_feature([]),
})),
example(features=features({
"val": feature(), # feature with nothing in it
# missing idx feature
})),
example(features=features({
"val": float_feature([1, 2, -1]),
"idx0": int64_feature([0, 9, 3]), # unsorted
"idx1": int64_feature([1, 0, 2]),
}))
]
expected_outputs = [{
"sp": (np.array([[5, 0], [10, 2]], dtype=np.int64),
np.array([3.0, 4.0], dtype=np.float32),
np.array([13, 3], dtype=np.int64))
}, {
"sp": empty_sparse(np.float32, shape=[13, 3])
}, {
"sp": empty_sparse(np.float32, shape=[13, 3])
}, {
"sp": (np.array([[0, 1], [3, 2], [9, 0]], dtype=np.int64),
np.array([1.0, -1.0, 2.0], dtype=np.float32),
np.array([13, 3], dtype=np.int64))
}]
for proto, expected_output in zip(original, expected_outputs):
self._test({
"serialized": ops.convert_to_tensor(proto.SerializeToString()),
"features": {
"sp":
parsing_ops.SparseFeature(["idx0", "idx1"], "val",
dtypes.float32, [13, 3])
}
}, expected_output)
def testSerializedContainingDense(self):
aname = "a"
bname = "b*has+a:tricky_name"
original = [
example(features=features({
aname: float_feature([1, 1]),
bname: bytes_feature([b"b0_str"]),
})), example(features=features({
aname: float_feature([-1, -1]),
bname: bytes_feature([b""]),
}))
]
# pylint: disable=too-many-function-args
expected_outputs = [
{
aname:
np.array([1, 1], dtype=np.float32).reshape(1, 2, 1),
bname:
np.array(["b0_str"], dtype=bytes).reshape(
1, 1, 1, 1)
},
{
aname:
np.array([-1, -1], dtype=np.float32).reshape(1, 2, 1),
bname:
np.array([""], dtype=bytes).reshape(
1, 1, 1, 1)
}
]
# pylint: enable=too-many-function-args
for proto, expected_output in zip(original, expected_outputs):
# No defaults, values required
self._test({
"serialized": ops.convert_to_tensor(proto.SerializeToString()),
"features": {
aname:
parsing_ops.FixedLenFeature((1, 2, 1), dtype=dtypes.float32),
bname:
parsing_ops.FixedLenFeature(
(1, 1, 1, 1), dtype=dtypes.string),
}
}, expected_output)
# This test is identical as the previous one except
# for the creation of 'serialized'.
def testSerializedContainingDenseWithConcat(self):
aname = "a"
bname = "b*has+a:tricky_name"
# TODO(lew): Feature appearing twice should be an error in future.
original = [
(example(features=features({
aname: float_feature([10, 10]),
})), example(features=features({
aname: float_feature([1, 1]),
bname: bytes_feature([b"b0_str"]),
}))),
(
example(features=features({
bname: bytes_feature([b"b100"]),
})),
example(features=features({
aname: float_feature([-1, -1]),
bname: bytes_feature([b"b1"]),
})),),
]
# pylint: disable=too-many-function-args
expected_outputs = [
{
aname:
np.array([1, 1], dtype=np.float32).reshape(1, 2, 1),
bname:
np.array(["b0_str"], dtype=bytes).reshape(
1, 1, 1, 1)
},
{
aname:
np.array([-1, -1], dtype=np.float32).reshape(1, 2, 1),
bname:
np.array(["b1"], dtype=bytes).reshape(
1, 1, 1, 1)
}
]
# pylint: enable=too-many-function-args
for (m, n), expected_output in zip(original, expected_outputs):
# No defaults, values required
self._test({
"serialized":
ops.convert_to_tensor(
m.SerializeToString() + n.SerializeToString()),
"features": {
aname:
parsing_ops.FixedLenFeature((1, 2, 1), dtype=dtypes.float32),
bname:
parsing_ops.FixedLenFeature(
(1, 1, 1, 1), dtype=dtypes.string),
}
}, expected_output)
def testSerializedContainingDenseScalar(self):
original = [
example(features=features({
"a": float_feature([1]),
})), example(features=features({}))
]
expected_outputs = [{
"a": np.array([1], dtype=np.float32)
}, {
"a": np.array([-1], dtype=np.float32)
}]
for proto, expected_output in zip(original, expected_outputs):
self._test({
"serialized": ops.convert_to_tensor(proto.SerializeToString()),
"features": {
"a":
parsing_ops.FixedLenFeature(
(1,), dtype=dtypes.float32, default_value=-1),
}
}, expected_output)
def testSerializedContainingDenseWithDefaults(self):
original = [
example(features=features({
"a": float_feature([1, 1]),
})),
example(features=features({
"b": bytes_feature([b"b1"]),
})),
example(features=features({
"b": feature()
})),
]
# pylint: disable=too-many-function-args
expected_outputs = [
{
"a":
np.array([1, 1], dtype=np.float32).reshape(1, 2, 1),
"b":
np.array("tmp_str", dtype=bytes).reshape(
1, 1, 1, 1)
},
{
"a":
np.array([3, -3], dtype=np.float32).reshape(1, 2, 1),
"b":
np.array("b1", dtype=bytes).reshape(
1, 1, 1, 1)
},
{
"a":
np.array([3, -3], dtype=np.float32).reshape(1, 2, 1),
"b":
np.array("tmp_str", dtype=bytes).reshape(
1, 1, 1, 1)
}
]
# pylint: enable=too-many-function-args
for proto, expected_output in zip(original, expected_outputs):
self._test({
"serialized": ops.convert_to_tensor(proto.SerializeToString()),
"features": {
"a":
parsing_ops.FixedLenFeature(
(1, 2, 1),
dtype=dtypes.float32,
default_value=[3.0, -3.0]),
"b":
parsing_ops.FixedLenFeature(
(1, 1, 1, 1),
dtype=dtypes.string,
default_value="tmp_str"),
}
}, expected_output)
@test_util.run_deprecated_v1
def testSerializedContainingSparseAndSparseFeatureAndDenseWithNoDefault(self):
original = [
example(features=features({
"c": float_feature([3, 4]),
"val": bytes_feature([b"a", b"b"]),
"idx": int64_feature([0, 3])
})), example(features=features({
"c": float_feature([1, 2]),
"val": bytes_feature([b"c"]),
"idx": int64_feature([7])
}))
]
a_default = np.array([[1, 2, 3]], dtype=np.int64)
b_default = np.random.rand(3, 3).astype(bytes)
expected_st_a = empty_sparse(np.int64)
expected_outputs = [{
"st_a":
expected_st_a,
"sp": (np.array([[0], [3]], dtype=np.int64),
np.array(["a", "b"], dtype=bytes), np.array(
[13], dtype=np.int64)),
"a":
a_default,
"b":
b_default,
"c":
np.array([3, 4], dtype=np.float32)
}, {
"st_a":
expected_st_a,
"sp": (np.array([[7]], dtype=np.int64), np.array(["c"], dtype=bytes),
np.array([13], dtype=np.int64)),
"a":
a_default,
"b":
b_default,
"c":
np.array([1, 2], dtype=np.float32)
}]
for proto, expected_output in zip(original, expected_outputs):
self._test(
{
"serialized": ops.convert_to_tensor(proto.SerializeToString()),
"features": {
"st_a":
parsing_ops.VarLenFeature(dtypes.int64),
"sp":
parsing_ops.SparseFeature("idx", "val", dtypes.string, 13
),
"a":
parsing_ops.FixedLenFeature(
(1, 3), dtypes.int64, default_value=a_default),
"b":
parsing_ops.FixedLenFeature(
(3, 3), dtypes.string, default_value=b_default),
# Feature "c" must be provided, since it has no default_value.
"c":
parsing_ops.FixedLenFeature((2,), dtypes.float32),
}
},
expected_output)
@test_util.run_deprecated_v1
def testSerializedContainingSparseAndSparseFeatureWithReuse(self):
original = [
example(features=features({
"val": bytes_feature([b"a", b"b"]),
"idx": int64_feature([0, 3])
})), example(features=features({
"val": bytes_feature([b"c", b"d"]),
"idx": int64_feature([7, 1])
}))
]
expected_outputs = [{
"idx": (np.array([[0], [1]], dtype=np.int64),
np.array([0, 3], dtype=np.int64), np.array([2],
dtype=np.int64)),
"sp": (np.array([[0], [3]], dtype=np.int64),
np.array(["a", "b"], dtype=bytes), np.array(
[13], dtype=np.int64))
},
{
"idx": (np.array([[0], [1]], dtype=np.int64),
np.array([7, 1], dtype=np.int64),
np.array([2], dtype=np.int64)),
"sp": (np.array([[1], [7]], dtype=np.int64),
np.array(["d", "c"], dtype=bytes),
np.array([13], dtype=np.int64))
}]
for proto, expected_output in zip(original, expected_outputs):
self._test({
"serialized": ops.convert_to_tensor(proto.SerializeToString()),
"features": {
"idx":
parsing_ops.VarLenFeature(dtypes.int64),
"sp":
parsing_ops.SparseFeature(["idx"], "val", dtypes.string, [13]
),
}
}, expected_output)
@test_util.run_deprecated_v1
def testSerializedContainingVarLenDense(self):
aname = "a"
bname = "b"
cname = "c"
dname = "d"
original = [
example(features=features({
cname: int64_feature([2]),
})),
example(features=features({
aname: float_feature([1, 1]),
bname: bytes_feature([b"b0_str", b"b1_str"]),
})),
example(features=features({
aname: float_feature([-1, -1, 2, 2]),
bname: bytes_feature([b"b1"]),
})),
example(features=features({
aname: float_feature([]),
cname: int64_feature([3]),
})),
]
# pylint: disable=too-many-function-args
expected_outputs = [
{
aname: np.empty(shape=(0, 2, 1), dtype=np.int64),
bname: np.empty(shape=(0, 1, 1, 1), dtype=bytes),
cname: np.array([2], dtype=np.int64),
dname: np.empty(shape=(0,), dtype=bytes)
},
{
aname:
np.array([[[1], [1]]], dtype=np.float32),
bname:
np.array(["b0_str", "b1_str"], dtype=bytes).reshape(2, 1, 1, 1),
cname:
np.empty(shape=(0,), dtype=np.int64),
dname:
np.empty(shape=(0,), dtype=bytes)
},
{
aname: np.array([[[-1], [-1]], [[2], [2]]], dtype=np.float32),
bname: np.array(["b1"], dtype=bytes).reshape(1, 1, 1, 1),
cname: np.empty(shape=(0,), dtype=np.int64),
dname: np.empty(shape=(0,), dtype=bytes)
},
{
aname: np.empty(shape=(0, 2, 1), dtype=np.int64),
bname: np.empty(shape=(0, 1, 1, 1), dtype=bytes),
cname: np.array([3], dtype=np.int64),
dname: np.empty(shape=(0,), dtype=bytes)
},
]
# pylint: enable=too-many-function-args
for proto, expected_output in zip(original, expected_outputs):
self._test({
"serialized": ops.convert_to_tensor(proto.SerializeToString()),
"features": {
aname:
parsing_ops.FixedLenSequenceFeature(
(2, 1), dtype=dtypes.float32, allow_missing=True),
bname:
parsing_ops.FixedLenSequenceFeature(
(1, 1, 1), dtype=dtypes.string, allow_missing=True),
cname:
parsing_ops.FixedLenSequenceFeature(
shape=[], dtype=dtypes.int64, allow_missing=True),
dname:
parsing_ops.FixedLenSequenceFeature(
shape=[], dtype=dtypes.string, allow_missing=True),
}
}, expected_output)
# Test with padding values.
# NOTE(mrry): Since we parse a single example at a time, the fixed-length
# sequences will not be padded, and the padding value will be ignored.
for proto, expected_output in zip(original, expected_outputs):
self._test({
"serialized": ops.convert_to_tensor(proto.SerializeToString()),
"features": {
aname:
parsing_ops.FixedLenSequenceFeature(
(2, 1), dtype=dtypes.float32, allow_missing=True),
bname:
parsing_ops.FixedLenSequenceFeature(
(1, 1, 1), dtype=dtypes.string, allow_missing=True),
cname:
parsing_ops.FixedLenSequenceFeature(
shape=[], dtype=dtypes.int64, allow_missing=True),
dname:
parsing_ops.FixedLenSequenceFeature(
shape=[], dtype=dtypes.string, allow_missing=True),
}
}, expected_output)
# Change number of required values so the inputs are not a
# multiple of this size.
self._test(
{
"serialized":
ops.convert_to_tensor(original[2].SerializeToString()),
"features": {
aname:
parsing_ops.FixedLenSequenceFeature(
(2, 1), dtype=dtypes.float32, allow_missing=True),
bname:
parsing_ops.FixedLenSequenceFeature(
(2, 1, 1), dtype=dtypes.string, allow_missing=True),
}
},
# TODO(mrry): Consider matching the `io.parse_example()` error message.
expected_err=(errors_impl.OpError, "key b:"))
self._test(
{
"serialized": ops.convert_to_tensor(""),
"features": {
aname:
parsing_ops.FixedLenSequenceFeature(
(2, 1),
dtype=dtypes.float32,
allow_missing=True,
default_value=[]),
bname:
parsing_ops.FixedLenSequenceFeature(
(2, 1, 1), dtype=dtypes.string, allow_missing=True),
}
},
expected_err=(ValueError,
"Cannot reshape a tensor with 0 elements to shape"))
self._test(
{
"serialized": ops.convert_to_tensor(""),
"features": {
aname:
parsing_ops.FixedLenFeature(
(None, 2, 1), dtype=dtypes.float32),
bname:
parsing_ops.FixedLenSequenceFeature(
(2, 1, 1), dtype=dtypes.string, allow_missing=True),
}
},
expected_err=(ValueError,
"First dimension of shape for feature a unknown. "
"Consider using FixedLenSequenceFeature."))
self._test(
{
"serialized": ops.convert_to_tensor(""),
"features": {
cname:
parsing_ops.FixedLenFeature(
(1, None), dtype=dtypes.int64, default_value=[[1]]),
}
},
expected_err=(ValueError,
"All dimensions of shape for feature c need to be known "
r"but received \(1, None\)."))
self._test(
{
"serialized": ops.convert_to_tensor(""),
"features": {
aname:
parsing_ops.FixedLenSequenceFeature(
(2, 1), dtype=dtypes.float32, allow_missing=True),
bname:
parsing_ops.FixedLenSequenceFeature(
(1, 1, 1), dtype=dtypes.string, allow_missing=True),
cname:
parsing_ops.FixedLenSequenceFeature(
shape=[], dtype=dtypes.int64, allow_missing=False),
dname:
parsing_ops.FixedLenSequenceFeature(
shape=[], dtype=dtypes.string, allow_missing=True),
}
},
expected_err=(ValueError,
"Unsupported: FixedLenSequenceFeature requires "
"allow_missing to be True."))
class ParseSingleExampleTest(test.TestCase):
def _test(self, kwargs, expected_values=None, expected_err=None):
with self.cached_session() as sess:
if expected_err:
with self.assertRaisesWithPredicateMatch(expected_err[0],
expected_err[1]):
out = parsing_ops.parse_single_example(**kwargs)
sess.run(flatten_values_tensors_or_sparse(out.values()))
return
else:
# Returns dict w/ Tensors and SparseTensors.
out = parsing_ops.parse_single_example(**kwargs)
# Check values.
tf_result = sess.run(flatten_values_tensors_or_sparse(out.values()))
_compare_output_to_expected(self, out, expected_values, tf_result)
# Check shapes.
for k, f in kwargs["features"].items():
if isinstance(f, parsing_ops.FixedLenFeature) and f.shape is not None:
self.assertEqual(tuple(out[k].get_shape()),
tensor_shape.as_shape(f.shape))
elif isinstance(f, parsing_ops.VarLenFeature):
self.assertEqual(
tuple(out[k].indices.get_shape().as_list()), (None, 1))
self.assertEqual(tuple(out[k].values.get_shape().as_list()), (None,))
self.assertEqual(
tuple(out[k].dense_shape.get_shape().as_list()), (1,))
@test_util.run_deprecated_v1
def testSingleExampleWithSparseAndSparseFeatureAndDense(self):
original = example(features=features({
"c": float_feature([3, 4]),
"d": float_feature([0.0, 1.0]),
"val": bytes_feature([b"a", b"b"]),
"idx": int64_feature([0, 3]),
"st_a": float_feature([3.0, 4.0])
}))
serialized = original.SerializeToString()
expected_st_a = (
np.array(
[[0], [1]], dtype=np.int64), # indices
np.array(
[3.0, 4.0], dtype=np.float32), # values
np.array(
[2], dtype=np.int64)) # shape: max_values = 2
expected_sp = ( # indices, values, shape
np.array(
[[0], [3]], dtype=np.int64), np.array(
["a", "b"], dtype="|S"), np.array(
[13], dtype=np.int64)) # max_values = 13
a_default = [1, 2, 3]
b_default = np.random.rand(3, 3).astype(bytes)
expected_output = {
"st_a": expected_st_a,
"sp": expected_sp,
"a": [a_default],
"b": b_default,
"c": np.array([3, 4], dtype=np.float32),
"d": np.array([0.0, 1.0], dtype=np.float32),
}
self._test(
{
"serialized":
ops.convert_to_tensor(serialized),
"features": {
"st_a":
parsing_ops.VarLenFeature(dtypes.float32),
"sp":
parsing_ops.SparseFeature(
["idx"], "val", dtypes.string, [13]),
"a":
parsing_ops.FixedLenFeature(
(1, 3), dtypes.int64, default_value=a_default),
"b":
parsing_ops.FixedLenFeature(
(3, 3), dtypes.string, default_value=b_default),
# Feature "c" must be provided, since it has no default_value.
"c":
parsing_ops.FixedLenFeature(2, dtypes.float32),
"d":
parsing_ops.FixedLenSequenceFeature([],
dtypes.float32,
allow_missing=True)
}
},
expected_output)
def testExampleLongerThanSpec(self):
serialized = example(
features=features({
"a": bytes_feature([b"a", b"b"]),
})).SerializeToString()
self._test(
{
"serialized": ops.convert_to_tensor(serialized),
"features": {
"a": parsing_ops.FixedLenFeature(1, dtypes.string)
}
},
expected_err=(errors_impl.OpError, "Can't parse serialized Example"))
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/io_ops/parsing_ops_test.py
+2565
View File
File diff suppressed because it is too large Load Diff
tensorflow/python/kernel_tests/io_ops/reader_ops_test.py
+745
View File
@@ -0,0 +1,745 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for Reader ops from io_ops."""
import collections
import gzip
import os
import threading
import zlib
from tensorflow.core.protobuf import config_pb2
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import test_util
from tensorflow.python.lib.io import tf_record
from tensorflow.python.ops import data_flow_ops
from tensorflow.python.ops import io_ops
from tensorflow.python.ops import variables
from tensorflow.python.platform import test
from tensorflow.python.util import compat
prefix_path = "tensorflow/core/lib"
# pylint: disable=invalid-name
TFRecordCompressionType = tf_record.TFRecordCompressionType
# pylint: enable=invalid-name
# Edgar Allan Poe's 'Eldorado'
_TEXT = b"""Gaily bedight,
A gallant knight,
In sunshine and in shadow,
Had journeyed long,
Singing a song,
In search of Eldorado.
But he grew old
This knight so bold
And o'er his heart a shadow
Fell as he found
No spot of ground
That looked like Eldorado.
And, as his strength
Failed him at length,
He met a pilgrim shadow
'Shadow,' said he,
'Where can it be
This land of Eldorado?'
'Over the Mountains
Of the Moon'
Down the Valley of the Shadow,
Ride, boldly ride,'
The shade replied,
'If you seek for Eldorado!'
"""
class TFCompressionTestCase(test.TestCase):
def setUp(self):
super(TFCompressionTestCase, self).setUp()
self._num_files = 2
self._num_records = 7
def _Record(self, f, r):
return compat.as_bytes("Record %d of file %d" % (r, f))
def _CreateFiles(self, options=None, prefix=""):
filenames = []
for i in range(self._num_files):
name = prefix + "tfrecord.%d.txt" % i
records = [self._Record(i, j) for j in range(self._num_records)]
fn = self._WriteRecordsToFile(records, name, options)
filenames.append(fn)
return filenames
def _WriteRecordsToFile(self, records, name="tfrecord", options=None):
fn = os.path.join(self.get_temp_dir(), name)
with tf_record.TFRecordWriter(fn, options=options) as writer:
for r in records:
writer.write(r)
return fn
def _ZlibCompressFile(self, infile, name="tfrecord.z"):
# zlib compress the file and write compressed contents to file.
with open(infile, "rb") as f:
cdata = zlib.compress(f.read())
zfn = os.path.join(self.get_temp_dir(), name)
with open(zfn, "wb") as f:
f.write(cdata)
return zfn
def _GzipCompressFile(self, infile, name="tfrecord.gz"):
# gzip compress the file and write compressed contents to file.
with open(infile, "rb") as f:
cdata = f.read()
gzfn = os.path.join(self.get_temp_dir(), name)
with gzip.GzipFile(gzfn, "wb") as f:
f.write(cdata)
return gzfn
def _ZlibDecompressFile(self, infile, name="tfrecord"):
with open(infile, "rb") as f:
cdata = zlib.decompress(f.read())
fn = os.path.join(self.get_temp_dir(), name)
with open(fn, "wb") as f:
f.write(cdata)
return fn
def _GzipDecompressFile(self, infile, name="tfrecord"):
with gzip.GzipFile(infile, "rb") as f:
cdata = f.read()
fn = os.path.join(self.get_temp_dir(), name)
with open(fn, "wb") as f:
f.write(cdata)
return fn
class IdentityReaderTest(test.TestCase):
def _ExpectRead(self, key, value, expected):
k, v = self.evaluate([key, value])
self.assertAllEqual(expected, k)
self.assertAllEqual(expected, v)
@test_util.run_deprecated_v1
def testOneEpoch(self):
reader = io_ops.IdentityReader("test_reader")
work_completed = reader.num_work_units_completed()
produced = reader.num_records_produced()
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
queued_length = queue.size()
key, value = reader.read(queue)
self.assertAllEqual(0, self.evaluate(work_completed))
self.assertAllEqual(0, self.evaluate(produced))
self.assertAllEqual(0, self.evaluate(queued_length))
self.evaluate(queue.enqueue_many([["A", "B", "C"]]))
self.evaluate(queue.close())
self.assertAllEqual(3, self.evaluate(queued_length))
self._ExpectRead(key, value, b"A")
self.assertAllEqual(1, self.evaluate(produced))
self._ExpectRead(key, value, b"B")
self._ExpectRead(key, value, b"C")
self.assertAllEqual(3, self.evaluate(produced))
self.assertAllEqual(0, self.evaluate(queued_length))
with self.assertRaisesOpError("is closed and has insufficient elements "
"\\(requested 1, current size 0\\)"):
self.evaluate([key, value])
self.assertAllEqual(3, self.evaluate(work_completed))
self.assertAllEqual(3, self.evaluate(produced))
self.assertAllEqual(0, self.evaluate(queued_length))
@test_util.run_deprecated_v1
def testMultipleEpochs(self):
reader = io_ops.IdentityReader("test_reader")
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
enqueue = queue.enqueue_many([["DD", "EE"]])
key, value = reader.read(queue)
self.evaluate(enqueue)
self._ExpectRead(key, value, b"DD")
self._ExpectRead(key, value, b"EE")
self.evaluate(enqueue)
self._ExpectRead(key, value, b"DD")
self._ExpectRead(key, value, b"EE")
self.evaluate(enqueue)
self._ExpectRead(key, value, b"DD")
self._ExpectRead(key, value, b"EE")
self.evaluate(queue.close())
with self.assertRaisesOpError("is closed and has insufficient elements "
"\\(requested 1, current size 0\\)"):
self.evaluate([key, value])
@test_util.run_deprecated_v1
def testSerializeRestore(self):
reader = io_ops.IdentityReader("test_reader")
produced = reader.num_records_produced()
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
self.evaluate(queue.enqueue_many([["X", "Y", "Z"]]))
key, value = reader.read(queue)
self._ExpectRead(key, value, b"X")
self.assertAllEqual(1, self.evaluate(produced))
state = self.evaluate(reader.serialize_state())
self._ExpectRead(key, value, b"Y")
self._ExpectRead(key, value, b"Z")
self.assertAllEqual(3, self.evaluate(produced))
self.evaluate(queue.enqueue_many([["Y", "Z"]]))
self.evaluate(queue.close())
self.evaluate(reader.restore_state(state))
self.assertAllEqual(1, self.evaluate(produced))
self._ExpectRead(key, value, b"Y")
self._ExpectRead(key, value, b"Z")
with self.assertRaisesOpError("is closed and has insufficient elements "
"\\(requested 1, current size 0\\)"):
self.evaluate([key, value])
self.assertAllEqual(3, self.evaluate(produced))
self.assertEqual(bytes, type(state))
with self.assertRaises(ValueError):
reader.restore_state([])
with self.assertRaises(ValueError):
reader.restore_state([state, state])
with self.assertRaisesOpError(
"Could not parse state for IdentityReader 'test_reader'"):
self.evaluate(reader.restore_state(state[1:]))
with self.assertRaisesOpError(
"Could not parse state for IdentityReader 'test_reader'"):
self.evaluate(reader.restore_state(state[:-1]))
with self.assertRaisesOpError(
"Could not parse state for IdentityReader 'test_reader'"):
self.evaluate(reader.restore_state(state + b"ExtraJunk"))
with self.assertRaisesOpError(
"Could not parse state for IdentityReader 'test_reader'"):
self.evaluate(reader.restore_state(b"PREFIX" + state))
with self.assertRaisesOpError(
"Could not parse state for IdentityReader 'test_reader'"):
self.evaluate(reader.restore_state(b"BOGUS" + state[5:]))
@test_util.run_deprecated_v1
def testReset(self):
reader = io_ops.IdentityReader("test_reader")
work_completed = reader.num_work_units_completed()
produced = reader.num_records_produced()
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
queued_length = queue.size()
key, value = reader.read(queue)
self.evaluate(queue.enqueue_many([["X", "Y", "Z"]]))
self._ExpectRead(key, value, b"X")
self.assertLess(0, self.evaluate(queued_length))
self.assertAllEqual(1, self.evaluate(produced))
self._ExpectRead(key, value, b"Y")
self.assertLess(0, self.evaluate(work_completed))
self.assertAllEqual(2, self.evaluate(produced))
self.evaluate(reader.reset())
self.assertAllEqual(0, self.evaluate(work_completed))
self.assertAllEqual(0, self.evaluate(produced))
self.assertAllEqual(1, self.evaluate(queued_length))
self._ExpectRead(key, value, b"Z")
self.evaluate(queue.enqueue_many([["K", "L"]]))
self._ExpectRead(key, value, b"K")
class WholeFileReaderTest(test.TestCase):
def setUp(self):
super(WholeFileReaderTest, self).setUp()
self._filenames = [
os.path.join(self.get_temp_dir(), "whole_file.%d.txt" % i)
for i in range(3)
]
self._content = [b"One\na\nb\n", b"Two\nC\nD", b"Three x, y, z"]
for fn, c in zip(self._filenames, self._content):
with open(fn, "wb") as h:
h.write(c)
def tearDown(self):
for fn in self._filenames:
os.remove(fn)
super(WholeFileReaderTest, self).tearDown()
def _ExpectRead(self, key, value, index):
k, v = self.evaluate([key, value])
self.assertAllEqual(compat.as_bytes(self._filenames[index]), k)
self.assertAllEqual(self._content[index], v)
@test_util.run_deprecated_v1
def testOneEpoch(self):
reader = io_ops.WholeFileReader("test_reader")
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
self.evaluate(queue.enqueue_many([self._filenames]))
self.evaluate(queue.close())
key, value = reader.read(queue)
self._ExpectRead(key, value, 0)
self._ExpectRead(key, value, 1)
self._ExpectRead(key, value, 2)
with self.assertRaisesOpError("is closed and has insufficient elements "
"\\(requested 1, current size 0\\)"):
self.evaluate([key, value])
@test_util.run_deprecated_v1
def testInfiniteEpochs(self):
reader = io_ops.WholeFileReader("test_reader")
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
enqueue = queue.enqueue_many([self._filenames])
key, value = reader.read(queue)
self.evaluate(enqueue)
self._ExpectRead(key, value, 0)
self._ExpectRead(key, value, 1)
self.evaluate(enqueue)
self._ExpectRead(key, value, 2)
self._ExpectRead(key, value, 0)
self._ExpectRead(key, value, 1)
self.evaluate(enqueue)
self._ExpectRead(key, value, 2)
self._ExpectRead(key, value, 0)
class TextLineReaderTest(test.TestCase):
def setUp(self):
super(TextLineReaderTest, self).setUp()
self._num_files = 2
self._num_lines = 5
def _LineText(self, f, l):
return compat.as_bytes("%d: %d" % (f, l))
def _CreateFiles(self, crlf=False):
filenames = []
for i in range(self._num_files):
fn = os.path.join(self.get_temp_dir(), "text_line.%d.txt" % i)
filenames.append(fn)
with open(fn, "wb") as f:
for j in range(self._num_lines):
f.write(self._LineText(i, j))
# Always include a newline after the record unless it is
# at the end of the file, in which case we include it sometimes.
if j + 1 != self._num_lines or i == 0:
f.write(b"\r\n" if crlf else b"\n")
return filenames
def _testOneEpoch(self, files):
reader = io_ops.TextLineReader(name="test_reader")
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
key, value = reader.read(queue)
self.evaluate(queue.enqueue_many([files]))
self.evaluate(queue.close())
for i in range(self._num_files):
for j in range(self._num_lines):
k, v = self.evaluate([key, value])
self.assertAllEqual("%s:%d" % (files[i], j + 1), compat.as_text(k))
self.assertAllEqual(self._LineText(i, j), v)
with self.assertRaisesOpError("is closed and has insufficient elements "
"\\(requested 1, current size 0\\)"):
k, v = self.evaluate([key, value])
@test_util.run_deprecated_v1
def testOneEpochLF(self):
self._testOneEpoch(self._CreateFiles(crlf=False))
@test_util.run_deprecated_v1
def testOneEpochCRLF(self):
self._testOneEpoch(self._CreateFiles(crlf=True))
@test_util.run_deprecated_v1
def testSkipHeaderLines(self):
files = self._CreateFiles()
reader = io_ops.TextLineReader(skip_header_lines=1, name="test_reader")
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
key, value = reader.read(queue)
self.evaluate(queue.enqueue_many([files]))
self.evaluate(queue.close())
for i in range(self._num_files):
for j in range(self._num_lines - 1):
k, v = self.evaluate([key, value])
self.assertAllEqual("%s:%d" % (files[i], j + 2), compat.as_text(k))
self.assertAllEqual(self._LineText(i, j + 1), v)
with self.assertRaisesOpError("is closed and has insufficient elements "
"\\(requested 1, current size 0\\)"):
k, v = self.evaluate([key, value])
class FixedLengthRecordReaderTest(TFCompressionTestCase):
def setUp(self):
super(FixedLengthRecordReaderTest, self).setUp()
self._num_files = 2
self._header_bytes = 5
self._record_bytes = 3
self._footer_bytes = 2
self._hop_bytes = 2
def _Record(self, f, r):
return compat.as_bytes(str(f * 2 + r) * self._record_bytes)
def _OverlappedRecord(self, f, r):
record_str = "".join([
str(i)[0]
for i in range(r * self._hop_bytes,
r * self._hop_bytes + self._record_bytes)
])
return compat.as_bytes(record_str)
# gap_bytes=hop_bytes-record_bytes
def _CreateFiles(self, num_records, gap_bytes):
filenames = []
for i in range(self._num_files):
fn = os.path.join(self.get_temp_dir(), "fixed_length_record.%d.txt" % i)
filenames.append(fn)
with open(fn, "wb") as f:
f.write(b"H" * self._header_bytes)
if num_records > 0:
f.write(self._Record(i, 0))
for j in range(1, num_records):
if gap_bytes > 0:
f.write(b"G" * gap_bytes)
f.write(self._Record(i, j))
f.write(b"F" * self._footer_bytes)
return filenames
def _CreateOverlappedRecordFiles(self, num_overlapped_records):
filenames = []
for i in range(self._num_files):
fn = os.path.join(self.get_temp_dir(),
"fixed_length_overlapped_record.%d.txt" % i)
filenames.append(fn)
with open(fn, "wb") as f:
f.write(b"H" * self._header_bytes)
if num_overlapped_records > 0:
all_records_str = "".join([
str(i)[0]
for i in range(self._record_bytes + self._hop_bytes *
(num_overlapped_records - 1))
])
f.write(compat.as_bytes(all_records_str))
f.write(b"F" * self._footer_bytes)
return filenames
# gap_bytes=hop_bytes-record_bytes
def _CreateGzipFiles(self, num_records, gap_bytes):
filenames = self._CreateFiles(num_records, gap_bytes)
for fn in filenames:
# compress inplace.
self._GzipCompressFile(fn, fn)
return filenames
# gap_bytes=hop_bytes-record_bytes
def _CreateZlibFiles(self, num_records, gap_bytes):
filenames = self._CreateFiles(num_records, gap_bytes)
for fn in filenames:
# compress inplace.
self._ZlibCompressFile(fn, fn)
return filenames
def _CreateGzipOverlappedRecordFiles(self, num_overlapped_records):
filenames = []
for i in range(self._num_files):
fn = os.path.join(self.get_temp_dir(),
"fixed_length_overlapped_record.%d.txt" % i)
filenames.append(fn)
with gzip.GzipFile(fn, "wb") as f:
f.write(b"H" * self._header_bytes)
if num_overlapped_records > 0:
all_records_str = "".join([
str(i)[0]
for i in range(self._record_bytes + self._hop_bytes *
(num_overlapped_records - 1))
])
f.write(compat.as_bytes(all_records_str))
f.write(b"F" * self._footer_bytes)
return filenames
def _CreateZlibOverlappedRecordFiles(self, num_overlapped_records):
filenames = []
for i in range(self._num_files):
fn = os.path.join(self.get_temp_dir(),
"fixed_length_overlapped_record.%d.txt" % i)
filenames.append(fn)
with open(fn + ".tmp", "wb") as f:
f.write(b"H" * self._header_bytes)
if num_overlapped_records > 0:
all_records_str = "".join([
str(i)[0]
for i in range(self._record_bytes + self._hop_bytes *
(num_overlapped_records - 1))
])
f.write(compat.as_bytes(all_records_str))
f.write(b"F" * self._footer_bytes)
self._ZlibCompressFile(fn + ".tmp", fn)
return filenames
# gap_bytes=hop_bytes-record_bytes
def _TestOneEpoch(self, files, num_records, gap_bytes, encoding=None):
hop_bytes = 0 if gap_bytes == 0 else self._record_bytes + gap_bytes
reader = io_ops.FixedLengthRecordReader(
header_bytes=self._header_bytes,
record_bytes=self._record_bytes,
footer_bytes=self._footer_bytes,
hop_bytes=hop_bytes,
encoding=encoding,
name="test_reader")
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
key, value = reader.read(queue)
self.evaluate(queue.enqueue_many([files]))
self.evaluate(queue.close())
for i in range(self._num_files):
for j in range(num_records):
k, v = self.evaluate([key, value])
self.assertAllEqual("%s:%d" % (files[i], j), compat.as_text(k))
self.assertAllEqual(self._Record(i, j), v)
with self.assertRaisesOpError("is closed and has insufficient elements "
"\\(requested 1, current size 0\\)"):
k, v = self.evaluate([key, value])
def _TestOneEpochWithHopBytes(self,
files,
num_overlapped_records,
encoding=None):
reader = io_ops.FixedLengthRecordReader(
header_bytes=self._header_bytes,
record_bytes=self._record_bytes,
footer_bytes=self._footer_bytes,
hop_bytes=self._hop_bytes,
encoding=encoding,
name="test_reader")
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
key, value = reader.read(queue)
self.evaluate(queue.enqueue_many([files]))
self.evaluate(queue.close())
for i in range(self._num_files):
for j in range(num_overlapped_records):
k, v = self.evaluate([key, value])
self.assertAllEqual("%s:%d" % (files[i], j), compat.as_text(k))
self.assertAllEqual(self._OverlappedRecord(i, j), v)
with self.assertRaisesOpError("is closed and has insufficient elements "
"\\(requested 1, current size 0\\)"):
k, v = self.evaluate([key, value])
@test_util.run_deprecated_v1
def testOneEpoch(self):
for num_records in [0, 7]:
# gap_bytes=0: hop_bytes=0
# gap_bytes=1: hop_bytes=record_bytes+1
for gap_bytes in [0, 1]:
files = self._CreateFiles(num_records, gap_bytes)
self._TestOneEpoch(files, num_records, gap_bytes)
@test_util.run_deprecated_v1
def testGzipOneEpoch(self):
for num_records in [0, 7]:
# gap_bytes=0: hop_bytes=0
# gap_bytes=1: hop_bytes=record_bytes+1
for gap_bytes in [0, 1]:
files = self._CreateGzipFiles(num_records, gap_bytes)
self._TestOneEpoch(files, num_records, gap_bytes, encoding="GZIP")
@test_util.run_deprecated_v1
def testZlibOneEpoch(self):
for num_records in [0, 7]:
# gap_bytes=0: hop_bytes=0
# gap_bytes=1: hop_bytes=record_bytes+1
for gap_bytes in [0, 1]:
files = self._CreateZlibFiles(num_records, gap_bytes)
self._TestOneEpoch(files, num_records, gap_bytes, encoding="ZLIB")
@test_util.run_deprecated_v1
def testOneEpochWithHopBytes(self):
for num_overlapped_records in [0, 2]:
files = self._CreateOverlappedRecordFiles(num_overlapped_records)
self._TestOneEpochWithHopBytes(files, num_overlapped_records)
@test_util.run_deprecated_v1
def testGzipOneEpochWithHopBytes(self):
for num_overlapped_records in [0, 2]:
files = self._CreateGzipOverlappedRecordFiles(num_overlapped_records,)
self._TestOneEpochWithHopBytes(
files, num_overlapped_records, encoding="GZIP")
@test_util.run_deprecated_v1
def testZlibOneEpochWithHopBytes(self):
for num_overlapped_records in [0, 2]:
files = self._CreateZlibOverlappedRecordFiles(num_overlapped_records)
self._TestOneEpochWithHopBytes(
files, num_overlapped_records, encoding="ZLIB")
class TFRecordReaderTest(TFCompressionTestCase):
def setUp(self):
super(TFRecordReaderTest, self).setUp()
@test_util.run_deprecated_v1
def testOneEpoch(self):
files = self._CreateFiles()
reader = io_ops.TFRecordReader(name="test_reader")
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
key, value = reader.read(queue)
self.evaluate(queue.enqueue_many([files]))
self.evaluate(queue.close())
for i in range(self._num_files):
for j in range(self._num_records):
k, v = self.evaluate([key, value])
self.assertTrue(compat.as_text(k).startswith("%s:" % files[i]))
self.assertAllEqual(self._Record(i, j), v)
with self.assertRaisesOpError("is closed and has insufficient elements "
"\\(requested 1, current size 0\\)"):
k, v = self.evaluate([key, value])
@test_util.run_deprecated_v1
def testReadUpTo(self):
files = self._CreateFiles()
reader = io_ops.TFRecordReader(name="test_reader")
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
batch_size = 3
key, value = reader.read_up_to(queue, batch_size)
self.evaluate(queue.enqueue_many([files]))
self.evaluate(queue.close())
num_k = 0
num_v = 0
while True:
try:
k, v = self.evaluate([key, value])
# Test reading *up to* batch_size records
self.assertLessEqual(len(k), batch_size)
self.assertLessEqual(len(v), batch_size)
num_k += len(k)
num_v += len(v)
except errors_impl.OutOfRangeError:
break
# Test that we have read everything
self.assertEqual(self._num_files * self._num_records, num_k)
self.assertEqual(self._num_files * self._num_records, num_v)
@test_util.run_deprecated_v1
def testReadZlibFiles(self):
options = tf_record.TFRecordOptions(TFRecordCompressionType.ZLIB)
files = self._CreateFiles(options)
reader = io_ops.TFRecordReader(name="test_reader", options=options)
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
key, value = reader.read(queue)
self.evaluate(queue.enqueue_many([files]))
self.evaluate(queue.close())
for i in range(self._num_files):
for j in range(self._num_records):
k, v = self.evaluate([key, value])
self.assertTrue(compat.as_text(k).startswith("%s:" % files[i]))
self.assertAllEqual(self._Record(i, j), v)
@test_util.run_deprecated_v1
def testReadGzipFiles(self):
options = tf_record.TFRecordOptions(TFRecordCompressionType.GZIP)
files = self._CreateFiles(options)
reader = io_ops.TFRecordReader(name="test_reader", options=options)
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
key, value = reader.read(queue)
self.evaluate(queue.enqueue_many([files]))
self.evaluate(queue.close())
for i in range(self._num_files):
for j in range(self._num_records):
k, v = self.evaluate([key, value])
self.assertTrue(compat.as_text(k).startswith("%s:" % files[i]))
self.assertAllEqual(self._Record(i, j), v)
class AsyncReaderTest(test.TestCase):
@test_util.run_deprecated_v1
def testNoDeadlockFromQueue(self):
"""Tests that reading does not block main execution threads."""
config = config_pb2.ConfigProto(
inter_op_parallelism_threads=1, intra_op_parallelism_threads=1)
with self.session(config=config) as sess:
thread_data_t = collections.namedtuple("thread_data_t",
["thread", "queue", "output"])
thread_data = []
# Create different readers, each with its own queue.
for i in range(3):
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
reader = io_ops.TextLineReader()
_, line = reader.read(queue)
output = []
t = threading.Thread(
target=AsyncReaderTest._RunSessionAndSave,
args=(sess, [line], output))
thread_data.append(thread_data_t(t, queue, output))
# Start all readers. They are all blocked waiting for queue entries.
self.evaluate(variables.global_variables_initializer())
for d in thread_data:
d.thread.start()
# Unblock the readers.
for i, d in enumerate(reversed(thread_data)):
fname = os.path.join(self.get_temp_dir(), "deadlock.%s.txt" % i)
with open(fname, "wb") as f:
f.write(("file-%s" % i).encode())
self.evaluate(d.queue.enqueue_many([[fname]]))
d.thread.join()
self.assertEqual([[("file-%s" % i).encode()]], d.output)
@staticmethod
def _RunSessionAndSave(sess, args, output):
output.append(sess.run(args))
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/io_ops/record_input_test.py
+198
View File
@@ -0,0 +1,198 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for record_input_op."""
import os
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import test_util
from tensorflow.python.lib.io import tf_record
from tensorflow.python.ops import data_flow_ops
from tensorflow.python.ops import variables
from tensorflow.python.platform import test
class RecordInputOpTest(test.TestCase):
def generateTestData(self,
prefix,
n,
m,
compression_type=tf_record.TFRecordCompressionType.NONE):
options = tf_record.TFRecordOptions(compression_type)
for i in range(n):
f = os.path.join(self.get_temp_dir(), prefix + "." + str(i))
w = tf_record.TFRecordWriter(f, options=options)
for j in range(m):
w.write("{0:0{width}}".format(i * m + j, width=10).encode("utf-8"))
w.close()
def testRecordInputSimple(self):
with self.cached_session() as sess:
self.generateTestData("basic", 1, 1)
yield_op = data_flow_ops.RecordInput(
file_pattern=os.path.join(self.get_temp_dir(), "basic.*"),
parallelism=1,
buffer_size=1,
batch_size=1,
name="record_input").get_yield_op()
self.assertEqual(self.evaluate(yield_op), b"0000000000")
def testRecordInputSimpleGzip(self):
with self.cached_session() as sess:
self.generateTestData(
"basic",
1,
1,
compression_type=tf_record.TFRecordCompressionType.GZIP)
yield_op = data_flow_ops.RecordInput(
file_pattern=os.path.join(self.get_temp_dir(), "basic.*"),
parallelism=1,
buffer_size=1,
batch_size=1,
name="record_input",
compression_type=tf_record.TFRecordCompressionType.GZIP).get_yield_op(
)
self.assertEqual(self.evaluate(yield_op), b"0000000000")
def testRecordInputSimpleZlib(self):
with self.cached_session() as sess:
self.generateTestData(
"basic",
1,
1,
compression_type=tf_record.TFRecordCompressionType.ZLIB)
yield_op = data_flow_ops.RecordInput(
file_pattern=os.path.join(self.get_temp_dir(), "basic.*"),
parallelism=1,
buffer_size=1,
batch_size=1,
name="record_input",
compression_type=tf_record.TFRecordCompressionType.ZLIB).get_yield_op(
)
self.assertEqual(self.evaluate(yield_op), b"0000000000")
@test_util.run_deprecated_v1
def testRecordInputEpochs(self):
files = 100
records_per_file = 100
batches = 2
with self.cached_session() as sess:
self.generateTestData("basic", files, records_per_file)
records = data_flow_ops.RecordInput(
file_pattern=os.path.join(self.get_temp_dir(), "basic.*"),
parallelism=2,
buffer_size=2000,
batch_size=1,
shift_ratio=0.33,
seed=10,
name="record_input",
batches=batches)
yield_op = records.get_yield_op()
# cycle over 3 epochs and make sure we never duplicate
for _ in range(3):
epoch_set = set()
for _ in range(int(files * records_per_file / batches)):
op_list = self.evaluate(yield_op)
self.assertTrue(len(op_list) is batches)
for r in op_list:
self.assertTrue(r[0] not in epoch_set)
epoch_set.add(r[0])
@test_util.run_deprecated_v1
def testDoesNotDeadlock(self):
# Iterate multiple times to cause deadlock if there is a chance it can occur
for _ in range(30):
with self.cached_session() as sess:
self.generateTestData("basic", 1, 1)
records = data_flow_ops.RecordInput(
file_pattern=os.path.join(self.get_temp_dir(), "basic.*"),
parallelism=1,
buffer_size=100,
batch_size=1,
name="record_input")
yield_op = records.get_yield_op()
for _ in range(50):
self.evaluate(yield_op)
@test_util.run_deprecated_v1
def testEmptyGlob(self):
with self.cached_session() as sess:
record_input = data_flow_ops.RecordInput(file_pattern="foo")
yield_op = record_input.get_yield_op()
self.evaluate(variables.global_variables_initializer())
with self.assertRaises(errors_impl.NotFoundError):
self.evaluate(yield_op)
@test_util.run_deprecated_v1
def testBufferTooSmall(self):
files = 10
records_per_file = 10
batches = 2
with self.cached_session() as sess:
self.generateTestData("basic", files, records_per_file)
records = data_flow_ops.RecordInput(
file_pattern=os.path.join(self.get_temp_dir(), "basic.*"),
parallelism=2,
buffer_size=2000,
batch_size=1,
shift_ratio=0.33,
seed=10,
name="record_input",
batches=batches)
yield_op = records.get_yield_op()
# cycle over 3 epochs and make sure we never duplicate
for _ in range(3):
epoch_set = set()
for _ in range(int(files * records_per_file / batches)):
op_list = self.evaluate(yield_op)
self.assertTrue(len(op_list) is batches)
for r in op_list:
self.assertTrue(r[0] not in epoch_set)
epoch_set.add(r[0])
def testInvalidParams(self):
with self.session():
with self.assertRaises(errors_impl.InvalidArgumentError):
self.evaluate(
data_flow_ops.gen_data_flow_ops.record_input(
file_pattern="nan",
file_buffer_size=-90,
file_parallelism=-438,
file_shuffle_shift_ratio=-784,
batch_size=-933,
file_random_seed=-678,
compression_type="nan",
)
)
if __name__ == "__main__":
test.main()
tensorflow/python/kernel_tests/io_ops/save_restore_ops_test.py
+107
View File
@@ -0,0 +1,107 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tensorflow.ops.io_ops."""
import os
from absl.testing import parameterized
from tensorflow.core.protobuf import config_pb2
from tensorflow.python.client import session
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import gen_io_ops
from tensorflow.python.ops import io_ops
from tensorflow.python.platform import test
_TEST_DTYPES = [dtypes.float32, dtypes.int32, dtypes.int4, dtypes.uint4]
class SaveRestoreTest(test.TestCase, parameterized.TestCase):
@parameterized.parameters(_TEST_DTYPES)
@test_util.run_in_graph_and_eager_modes
def testRelativePath(self, dtype):
os.chdir(self.get_temp_dir())
self.evaluate(
io_ops.save_v2(
"ckpt", ["x"], [""], [constant_op.constant(2, dtype=dtype)]
)
)
self.assertAllEqual(
[2], self.evaluate(io_ops.restore_v2("ckpt", ["x"], [""], [dtype]))
)
@parameterized.parameters(_TEST_DTYPES)
def testWithSliceInput(self, dtype):
os.chdir(self.get_temp_dir())
self.evaluate(
io_ops.save_v2(
"ckpt",
["x"],
[""],
[constant_op.constant([[1, 2, 3], [2, 3, 4]], dtype=dtype)],
)
)
self.assertAllEqual(
[[2], [3]],
self.evaluate(io_ops.restore_v2("ckpt", ["x"], ["2 3 -:1,1"], [dtype]))[
0
],
)
class ShardedFileOpsTest(test.TestCase):
def testShardedFileName(self):
with session.Session(
target="", config=config_pb2.ConfigProto(device_count={"CPU": 2})):
self.assertEqual(
gen_io_ops.sharded_filename("foo", 4, 100).eval(),
b"foo-00004-of-00100")
self.assertEqual(
gen_io_ops.sharded_filespec("foo", 100).eval(), b"foo-?????-of-00100")
class ShapeInferenceTest(test.TestCase, parameterized.TestCase):
@parameterized.parameters(_TEST_DTYPES)
def testRestoreV2WithSliceInput(self, dtype):
with ops.Graph().as_default():
op = io_ops.restore_v2(
"model", ["var1", "var2"], ["", "3 4 0,1:-"], [dtype, dtype]
)
self.assertEqual(2, len(op))
self.assertFalse(op[0].get_shape().is_fully_defined())
self.assertEqual([1, 4], op[1].get_shape())
@parameterized.parameters(_TEST_DTYPES)
def testRestoreV2NumSlicesNotMatch(self, dtype):
with ops.Graph().as_default():
with self.assertRaises(ValueError):
io_ops.restore_v2(
"model", ["var1", "var2", "var3"], ["", "3 4 0,1:-"], [dtype, dtype]
)
@parameterized.parameters(_TEST_DTYPES)
def testRestoreSlice(self, dtype):
with ops.Graph().as_default():
op = gen_io_ops.restore_slice("model", "var", "3 4 0,1:-", dtype)
self.assertEqual([1, 4], op.get_shape())
if __name__ == "__main__":
test.main()