wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions 3 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/python/autograph/operators/BUILD
+295
View File
@@ -0,0 +1,295 @@
load("@xla//third_party/rules_python/python:py_library.bzl", "py_library")
load("//tensorflow:pytype.default.bzl", "pytype_strict_library")
load("//tensorflow:tensorflow.bzl", "py_test")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
licenses = ["notice"],
)
py_library(
name = "py_builtins",
srcs = ["py_builtins.py"],
strict_deps = True,
visibility = ["//tensorflow:__subpackages__"],
deps = [
"//tensorflow/python/autograph/utils:tensors",
"//tensorflow/python/autograph/utils:type_registry",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:tensor_util",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:cond",
"//tensorflow/python/ops:control_flow_assert",
"//tensorflow/python/ops:list_ops",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:parsing_ops_gen",
"//tensorflow/python/ops:string_ops_gen",
],
)
py_library(
name = "exceptions",
srcs = ["exceptions.py"],
strict_deps = True,
visibility = ["//visibility:private"],
deps = [
"//tensorflow/python/framework:tensor_util",
"//tensorflow/python/ops:control_flow_assert",
"//tensorflow/python/util:tf_inspect",
],
)
py_library(
name = "__init__",
srcs = ["__init__.py"],
strict_deps = True,
visibility = ["//tensorflow:__subpackages__"],
deps = [
":conditional_expressions",
":control_flow",
":data_structures",
":exceptions",
":logical",
":py_builtins",
":slices",
":variables",
],
)
py_library(
name = "logical",
srcs = ["logical.py"],
strict_deps = True,
visibility = ["//visibility:private"],
deps = [
"//tensorflow/python/framework:tensor_util",
"//tensorflow/python/ops:cond",
"//tensorflow/python/ops:math_ops_gen",
],
)
py_library(
name = "variables",
srcs = ["variables.py"],
strict_deps = True,
visibility = ["//tensorflow:__subpackages__"],
)
py_library(
name = "data_structures",
srcs = ["data_structures.py"],
strict_deps = True,
visibility = ["//tensorflow:__subpackages__"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:tensor_util",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:cond",
"//tensorflow/python/ops:list_ops",
"//tensorflow/python/ops:tensor_array_ops",
],
)
py_library(
name = "conditional_expressions",
srcs = ["conditional_expressions.py"],
strict_deps = True,
visibility = ["//visibility:private"],
deps = [
":control_flow",
"//tensorflow/python/autograph/utils:tensors",
"//tensorflow/python/ops:cond",
],
)
py_library(
name = "control_flow",
srcs = ["control_flow.py"],
strict_deps = True,
visibility = ["//tensorflow:__subpackages__"],
deps = [
":py_builtins",
":variables",
"//tensorflow/python/autograph/utils:ag_logging",
"//tensorflow/python/autograph/utils:misc",
"//tensorflow/python/autograph/utils:tensors",
"//tensorflow/python/autograph/utils:type_registry",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:func_graph",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:tensor_conversion",
"//tensorflow/python/framework:tensor_shape",
"//tensorflow/python/framework:tensor_util",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:cond",
"//tensorflow/python/ops:control_flow_assert",
"//tensorflow/python/ops:control_flow_util",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:tensor_array_ops",
"//tensorflow/python/ops:while_loop",
"//tensorflow/python/types:distribute",
"//tensorflow/python/util:nest",
"//tensorflow/python/util:variable_utils",
"//third_party/py/numpy",
],
)
py_library(
name = "slices",
srcs = ["slices.py"],
strict_deps = True,
visibility = ["//visibility:private"],
deps = [
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:tensor_util",
"//tensorflow/python/ops:array_ops_gen",
"//tensorflow/python/ops:list_ops",
"//tensorflow/python/ops:string_ops_gen",
"//tensorflow/python/ops:tensor_array_ops",
],
)
py_test(
name = "data_structures_test",
srcs = ["data_structures_test.py"],
strict_deps = True,
deps = [
":data_structures",
#internal proto upb dep
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:tensor",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:list_ops",
"//tensorflow/python/ops:tensor_array_ops",
"//tensorflow/python/platform:client_testlib",
],
)
py_test(
name = "conditional_expressions_test",
srcs = ["conditional_expressions_test.py"],
strict_deps = True,
deps = [
":conditional_expressions",
#internal proto upb dep
"//tensorflow/python/eager:def_function",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
],
)
py_test(
name = "control_flow_test",
srcs = ["control_flow_test.py"],
strict_deps = True,
tags = [
"no_gpu", # b/127001953
],
deps = [
":control_flow",
":variables",
#internal proto upb dep
"//third_party/py/numpy",
"//tensorflow/python/autograph/utils:ag_logging",
"//tensorflow/python/autograph/utils:testing",
"//tensorflow/python/data/ops:dataset_ops",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:tensor_shape",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:control_flow_assert",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:math_ops_gen",
"//tensorflow/python/ops:random_ops",
"//tensorflow/python/ops/ragged:ragged_factory_ops",
"//tensorflow/python/platform:client_testlib",
],
)
py_test(
name = "exceptions_test",
srcs = ["exceptions_test.py"],
strict_deps = True,
deps = [
":exceptions",
#internal proto upb dep
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
],
)
py_test(
name = "logical_test",
srcs = ["logical_test.py"],
strict_deps = True,
deps = [
":logical",
#internal proto upb dep
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
],
)
py_test(
name = "py_builtins_test",
srcs = ["py_builtins_test.py"],
strict_deps = True,
deps = [
":data_structures",
":py_builtins",
#internal proto upb dep
"//tensorflow/python/autograph/core:converter",
"//tensorflow/python/autograph/core:function_wrappers",
"//tensorflow/python/data/ops:dataset_ops",
"//tensorflow/python/eager:def_function",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:tensor_array_ops",
"//tensorflow/python/platform:client_testlib",
],
)
py_test(
name = "slices_test",
srcs = ["slices_test.py"],
strict_deps = True,
deps = [
":slices",
#internal proto upb dep
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/ops:list_ops",
"//tensorflow/python/platform:client_testlib",
],
)
py_test(
name = "variables_test",
srcs = ["variables_test.py"],
strict_deps = True,
deps = [
":variables",
#internal proto upb dep
"//tensorflow/python/platform:client_testlib",
],
)
pytype_strict_library(
name = "dispatch_context",
srcs = ["dispatch_context.py"],
)
tensorflow/python/autograph/operators/__init__.py
+63
View File
@@ -0,0 +1,63 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""This module implements operators that AutoGraph overloads.
Note that "operator" is used loosely here, and includes control structures like
conditionals and loops, implemented in functional form, using for example
closures for the body.
"""
# Naming conventions:
# * operator names match the name usually used for the respective Python
# idiom; examples: for_stmt, list_append
# * operator arguments match either of:
# - the corresponding Python AST attribute (e.g. the condition of an if
# statement is called test) if the operator represents an AST construct
# - the names used in the Python docs, if the operator is a function (e.g.
# list_ and x for append, see
# https://docs.python.org/3.7/tutorial/datastructures.html)
#
# All operators may accept a final argument named "opts", of a type that
# subclasses namedtuple and contains any arguments that are only required
# for some specializations of the operator.
from tensorflow.python.autograph.operators.conditional_expressions import if_exp
from tensorflow.python.autograph.operators.control_flow import for_stmt
from tensorflow.python.autograph.operators.control_flow import if_stmt
from tensorflow.python.autograph.operators.control_flow import while_stmt
from tensorflow.python.autograph.operators.data_structures import list_append
from tensorflow.python.autograph.operators.data_structures import list_pop
from tensorflow.python.autograph.operators.data_structures import list_stack
from tensorflow.python.autograph.operators.data_structures import ListPopOpts
from tensorflow.python.autograph.operators.data_structures import ListStackOpts
from tensorflow.python.autograph.operators.data_structures import new_list
from tensorflow.python.autograph.operators.exceptions import assert_stmt
from tensorflow.python.autograph.operators.logical import and_
from tensorflow.python.autograph.operators.logical import eq
from tensorflow.python.autograph.operators.logical import not_
from tensorflow.python.autograph.operators.logical import not_eq
from tensorflow.python.autograph.operators.logical import or_
from tensorflow.python.autograph.operators.py_builtins import float_
from tensorflow.python.autograph.operators.py_builtins import int_
from tensorflow.python.autograph.operators.py_builtins import len_
from tensorflow.python.autograph.operators.py_builtins import print_
from tensorflow.python.autograph.operators.py_builtins import range_
from tensorflow.python.autograph.operators.slices import get_item
from tensorflow.python.autograph.operators.slices import GetItemOpts
from tensorflow.python.autograph.operators.slices import set_item
from tensorflow.python.autograph.operators.variables import ld
from tensorflow.python.autograph.operators.variables import ldu
from tensorflow.python.autograph.operators.variables import Undefined
from tensorflow.python.autograph.operators.variables import UndefinedReturnValue
tensorflow/python/autograph/operators/conditional_expressions.py
+52
View File
@@ -0,0 +1,52 @@
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Conditional expressions (e.g. the ternary if statement)."""
from tensorflow.python.autograph.operators import control_flow
from tensorflow.python.autograph.utils import tensors
from tensorflow.python.ops import cond as tf_cond
def if_exp(cond, if_true, if_false, expr_repr):
if tensors.is_dense_tensor(cond):
return _tf_if_exp(cond, if_true, if_false, expr_repr)
else:
return _py_if_exp(cond, if_true, if_false)
def _tf_if_exp(cond, if_true, if_false, expr_repr):
"""Overload of if_exp that stages a TF cond."""
# TODO(mdan): Use nonlocal once we no longer need to support py2.
true_val = []
false_val = []
def true_fn():
true_val.append(if_true())
if true_val and false_val:
control_flow.verify_single_cond_var(expr_repr, true_val[0], false_val[0])
return true_val[0]
def false_fn():
false_val.append(if_false())
if true_val and false_val:
control_flow.verify_single_cond_var(expr_repr, true_val[0], false_val[0])
return false_val[0]
return tf_cond.cond(cond, true_fn, false_fn)
def _py_if_exp(cond, if_true, if_false):
return if_true() if cond else if_false()
tensorflow/python/autograph/operators/conditional_expressions_test.py
+61
View File
@@ -0,0 +1,61 @@
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for conditional_expressions module."""
from tensorflow.python.autograph.operators import conditional_expressions
from tensorflow.python.eager import def_function
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import test_util
from tensorflow.python.platform import test
def _basic_expr(cond):
return conditional_expressions.if_exp(
cond,
lambda: constant_op.constant(1),
lambda: constant_op.constant(2),
'cond')
@test_util.run_all_in_graph_and_eager_modes
class IfExpTest(test.TestCase):
def test_tensor(self):
self.assertEqual(self.evaluate(_basic_expr(constant_op.constant(True))), 1)
self.assertEqual(self.evaluate(_basic_expr(constant_op.constant(False))), 2)
def test_tensor_mismatched_type(self):
# tf.function required because eager cond degenerates to Python if.
@def_function.function
def test_fn():
conditional_expressions.if_exp(
constant_op.constant(True), lambda: 1.0, lambda: 2, 'expr_repr')
with self.assertRaisesRegex(
TypeError,
"'expr_repr' has dtype float32 in the main.*int32 in the else"):
test_fn()
def test_python(self):
self.assertEqual(self.evaluate(_basic_expr(True)), 1)
self.assertEqual(self.evaluate(_basic_expr(False)), 2)
self.assertEqual(
conditional_expressions.if_exp(True, lambda: 1, lambda: 2, ''), 1)
self.assertEqual(
conditional_expressions.if_exp(False, lambda: 1, lambda: 2, ''), 2)
if __name__ == '__main__':
test.main()
tensorflow/python/autograph/operators/control_flow.py
+1270
View File
File diff suppressed because it is too large Load Diff
tensorflow/python/autograph/operators/control_flow_test.py
+1347
View File
File diff suppressed because it is too large Load Diff
tensorflow/python/autograph/operators/data_structures.py
+347
View File
@@ -0,0 +1,347 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Operators specific to data structures: list append, subscripts, etc."""
import collections
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import cond
from tensorflow.python.ops import list_ops
from tensorflow.python.ops import tensor_array_ops
# TODO(mdan): Once control flow supports objects, repackage as a class.
def new_list(iterable=None):
"""The list constructor.
Args:
iterable: Optional elements to fill the list with.
Returns:
A list-like object. The exact return value depends on the initial elements.
"""
if iterable:
elements = tuple(iterable)
else:
elements = ()
if elements:
# When the list contains elements, it is assumed to be a "Python" lvalue
# list.
return _py_list_new(elements)
return tf_tensor_list_new(elements)
def tf_tensor_array_new(elements, element_dtype=None, element_shape=None):
"""Overload of new_list that stages a Tensor list creation."""
elements = tuple(ops.convert_to_tensor(el) for el in elements)
all_dtypes = set(el.dtype for el in elements)
if len(all_dtypes) == 1:
inferred_dtype, = tuple(all_dtypes)
if element_dtype is not None and element_dtype != inferred_dtype:
raise ValueError(
'incompatible dtype; specified: {}, inferred from {}: {}'.format(
element_dtype, elements, inferred_dtype))
elif len(all_dtypes) > 1:
raise ValueError(
'TensorArray requires all elements to have the same dtype:'
' {}'.format(elements))
else:
if element_dtype is None:
raise ValueError('dtype is required to create an empty TensorArray')
all_shapes = set(tuple(el.shape.as_list()) for el in elements)
if len(all_shapes) == 1:
inferred_shape, = tuple(all_shapes)
if element_shape is not None and element_shape != inferred_shape:
raise ValueError(
'incompatible shape; specified: {}, inferred from {}: {}'.format(
element_shape, elements, inferred_shape))
elif len(all_shapes) > 1:
raise ValueError(
'TensorArray requires all elements to have the same shape:'
' {}'.format(elements))
# TODO(mdan): We may want to allow different shapes with infer_shape=False.
else:
inferred_shape = None
if element_dtype is None:
element_dtype = inferred_dtype
if element_shape is None:
element_shape = inferred_shape
l = tensor_array_ops.TensorArray(
dtype=element_dtype,
size=len(elements),
dynamic_size=True,
infer_shape=(element_shape is None),
element_shape=element_shape)
for i, el in enumerate(elements):
l = l.write(i, el)
return l
def tf_tensor_list_new(elements, element_dtype=None, element_shape=None):
"""Overload of new_list that stages a Tensor list creation."""
if tensor_util.is_tf_type(elements):
if element_shape is not None:
raise ValueError(
'element shape may not be specified when creating list from tensor')
element_shape = array_ops.shape(elements)[1:]
l = list_ops.tensor_list_from_tensor(elements, element_shape=element_shape)
return l
elements = tuple(ops.convert_to_tensor(el) for el in elements)
all_dtypes = set(el.dtype for el in elements)
if len(all_dtypes) == 1:
inferred_dtype = tuple(all_dtypes)[0]
if element_dtype is not None and element_dtype != inferred_dtype:
raise ValueError(
'incompatible dtype; specified: {}, inferred from {}: {}'.format(
element_dtype, elements, inferred_dtype))
elif all_dtypes:
# Heterogeneous lists are ok.
if element_dtype is not None:
raise ValueError(
'specified dtype {} is inconsistent with that of elements {}'.format(
element_dtype, elements))
inferred_dtype = dtypes.variant
else:
inferred_dtype = dtypes.variant
all_shapes = set(tuple(el.shape.as_list()) for el in elements)
if len(all_shapes) == 1:
inferred_shape = array_ops.shape(elements[0])
if element_shape is not None and element_shape != inferred_shape:
raise ValueError(
'incompatible shape; specified: {}, inferred from {}: {}'.format(
element_shape, elements, inferred_shape))
elif all_shapes:
# Heterogeneous lists are ok.
if element_shape is not None:
raise ValueError(
'specified shape {} is inconsistent with that of elements {}'.format(
element_shape, elements))
inferred_shape = constant_op.constant(-1) # unknown shape, by convention
else:
inferred_shape = constant_op.constant(-1) # unknown shape, by convention
if element_dtype is None:
element_dtype = inferred_dtype
if element_shape is None:
element_shape = inferred_shape
element_shape = ops.convert_to_tensor(element_shape, dtype=dtypes.int32)
l = list_ops.empty_tensor_list(
element_shape=element_shape, element_dtype=element_dtype)
for el in elements:
l = list_ops.tensor_list_push_back(l, el)
return l
def _py_list_new(elements):
"""Overload of new_list that creates a Python list."""
return list(elements)
def list_append(list_, x):
"""The list append function.
Note: it is unspecified where list_ will be mutated or not. If list_ is
a TensorFlow entity, it will not be typically mutated. If list_ is a plain
list, it will be. In general, if the list is mutated then the return value
should point to the original entity.
Args:
list_: An entity that supports append semantics.
x: The element to append.
Returns:
Same as list_, after the append was performed.
Raises:
ValueError: if list_ is not of a known list-like type.
"""
if isinstance(list_, tensor_array_ops.TensorArray):
return _tf_tensorarray_append(list_, x)
elif tensor_util.is_tf_type(list_):
if list_.dtype == dtypes.variant:
return _tf_tensor_list_append(list_, x)
else:
raise ValueError(
'tensor lists are expected to be Tensors with dtype=tf.variant,'
' instead found %s' % list_)
else:
return _py_list_append(list_, x)
def _tf_tensor_list_append(list_, x):
"""Overload of list_append that stages a Tensor list write."""
def empty_list_of_elements_like_x():
tensor_x = ops.convert_to_tensor(x)
return list_ops.empty_tensor_list(
element_shape=array_ops.shape(tensor_x),
element_dtype=tensor_x.dtype)
list_ = cond.cond(
list_ops.tensor_list_length(list_) > 0,
lambda: list_,
empty_list_of_elements_like_x,
)
return list_ops.tensor_list_push_back(list_, x)
def _tf_tensorarray_append(list_, x):
"""Overload of list_append that stages a TensorArray write."""
return list_.write(list_.size(), x)
def _py_list_append(list_, x):
"""Overload of list_append that executes a Python list append."""
# Revert to the original call.
list_.append(x)
return list_
class ListPopOpts(
collections.namedtuple('ListPopOpts', ('element_dtype', 'element_shape'))):
pass
def list_pop(list_, i, opts):
"""The list pop function.
Note: it is unspecified where list_ will be mutated or not. If list_ is
a TensorFlow entity, it will not be typically mutated. If list_ is a plain
list, it will be. In general, if the list is mutated then the return value
should point to the original entity.
Args:
list_: An entity that supports pop semantics.
i: Optional index to pop from. May be None.
opts: A ListPopOpts.
Returns:
Tuple (x, out_list_):
out_list_: same as list_, after the removal was performed.
x: the removed element value.
Raises:
ValueError: if list_ is not of a known list-like type or the operation is
not supported for that type.
"""
assert isinstance(opts, ListPopOpts)
if isinstance(list_, tensor_array_ops.TensorArray):
raise ValueError('TensorArray does not support item removal')
elif tensor_util.is_tf_type(list_):
if list_.dtype == dtypes.variant:
return _tf_tensor_list_pop(list_, i, opts)
else:
raise ValueError(
'tensor lists are expected to be Tensors with dtype=tf.variant,'
' instead found %s' % list_)
else:
return _py_list_pop(list_, i)
def _tf_tensor_list_pop(list_, i, opts):
"""Overload of list_pop that stages a Tensor list pop."""
if i is not None:
raise NotImplementedError('tensor lists only support removing from the end')
if opts.element_dtype is None:
raise ValueError('cannot pop from a list without knowing its element '
'type; use set_element_type to annotate it')
if opts.element_shape is None:
raise ValueError('cannot pop from a list without knowing its element '
'shape; use set_element_type to annotate it')
list_out, x = list_ops.tensor_list_pop_back(
list_, element_dtype=opts.element_dtype)
x.set_shape(opts.element_shape)
return list_out, x
def _py_list_pop(list_, i):
"""Overload of list_pop that executes a Python list append."""
if i is None:
x = list_.pop()
else:
x = list_.pop(i)
return list_, x
# TODO(mdan): Look into reducing duplication between all these containers.
class ListStackOpts(
collections.namedtuple('ListStackOpts',
('element_dtype', 'original_call'))):
pass
def list_stack(list_, opts):
"""The list stack function.
This does not have a direct correspondent in Python. The closest idiom to
this is tf.append or np.stack. It's different from those in the sense that it
accepts a Tensor list, rather than a list of tensors. It can also accept
TensorArray. When the target is anything else, the dispatcher will rely on
ctx.original_call for fallback.
Args:
list_: An entity that supports append semantics.
opts: A ListStackOpts object.
Returns:
The output of the stack operation, typically a Tensor.
"""
assert isinstance(opts, ListStackOpts)
if isinstance(list_, tensor_array_ops.TensorArray):
return _tf_tensorarray_stack(list_)
elif tensor_util.is_tf_type(list_):
if list_.dtype == dtypes.variant:
return _tf_tensor_list_stack(list_, opts)
else:
# No-op for primitive Tensor arguments.
return list_
else:
return _py_list_stack(list_, opts)
def _tf_tensorarray_stack(list_):
"""Overload of list_stack that stages a TensorArray stack."""
return list_.stack()
def _tf_tensor_list_stack(list_, opts):
"""Overload of list_stack that stages a Tensor list write."""
if opts.element_dtype is None:
raise ValueError('cannot stack a list without knowing its element type;'
' use set_element_type to annotate it')
return list_ops.tensor_list_stack(list_, element_dtype=opts.element_dtype)
def _py_list_stack(list_, opts):
"""Overload of list_stack that executes a Python list append."""
# Revert to the original call.
return opts.original_call(list_)
tensorflow/python/autograph/operators/data_structures_test.py
+183
View File
@@ -0,0 +1,183 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for data_structures module."""
from tensorflow.python.autograph.operators import data_structures
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import tensor
from tensorflow.python.framework import test_util
from tensorflow.python.ops import list_ops
from tensorflow.python.ops import tensor_array_ops
from tensorflow.python.platform import test
class ListTest(test.TestCase):
def test_new_list_empty(self):
l = data_structures.new_list()
# Can't evaluate an empty list.
# TODO(mdan): sess.run should allow tf.variant maybe?
self.assertTrue(isinstance(l, tensor.Tensor))
def test_new_list_tensor(self):
l = data_structures.new_list([3, 4, 5])
self.assertAllEqual(l, [3, 4, 5])
def test_tf_tensor_list_new(self):
l = data_structures.tf_tensor_list_new([3, 4, 5])
t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
with self.cached_session() as sess:
self.assertAllEqual(self.evaluate(t), [3, 4, 5])
def test_tf_tensor_list_new_empty(self):
l = data_structures.tf_tensor_list_new([],
element_dtype=dtypes.int32,
element_shape=())
t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
with self.cached_session() as sess:
self.assertAllEqual(self.evaluate(t), [])
def test_tf_tensor_list_new_from_tensor(self):
l = data_structures.tf_tensor_list_new(constant_op.constant([3, 4, 5]))
t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
with self.cached_session() as sess:
self.assertAllEqual(self.evaluate(t), [3, 4, 5])
@test_util.run_deprecated_v1
def test_tf_tensor_list_new_illegal_input(self):
with self.assertRaises(ValueError):
data_structures.tf_tensor_list_new([3, 4.0])
# TODO(mdan): It might make more sense to type cast in this case.
with self.assertRaises(ValueError):
data_structures.tf_tensor_list_new([3, 4], element_dtype=dtypes.float32)
# Tensor lists do support heterogeneous lists.
self.assertIsNot(data_structures.tf_tensor_list_new([3, [4, 5]]), None)
with self.assertRaises(ValueError):
data_structures.tf_tensor_list_new([3, 4], element_shape=(2,))
with self.assertRaises(ValueError):
data_structures.tf_tensor_list_new(
constant_op.constant([1, 2, 3]), element_shape=[1])
def test_tf_tensor_array_new(self):
l = data_structures.tf_tensor_array_new([3, 4, 5])
t = l.stack()
with self.cached_session() as sess:
self.assertAllEqual(self.evaluate(t), [3, 4, 5])
def test_tf_tensor_array_new_illegal_input(self):
with self.assertRaises(ValueError):
data_structures.tf_tensor_array_new([3, 4.0])
with self.assertRaises(ValueError):
data_structures.tf_tensor_array_new([3, 4], element_dtype=dtypes.float32)
with self.assertRaises(ValueError):
data_structures.tf_tensor_array_new([3, [4, 5]])
with self.assertRaises(ValueError):
data_structures.tf_tensor_array_new([3, 4], element_shape=(2,))
with self.assertRaises(ValueError):
data_structures.tf_tensor_array_new([], element_shape=(2,))
# TAs can infer the shape.
self.assertIsNot(
data_structures.tf_tensor_array_new([], element_dtype=dtypes.float32),
None)
def test_append_tensor_list(self):
l = data_structures.new_list()
x = constant_op.constant([1, 2, 3])
l = data_structures.list_append(l, x)
t = list_ops.tensor_list_stack(l, element_dtype=x.dtype)
with self.cached_session() as sess:
self.assertAllEqual(self.evaluate(t), [[1, 2, 3]])
@test_util.run_deprecated_v1
def test_append_tensorarray(self):
l = tensor_array_ops.TensorArray(dtypes.int32, size=0, dynamic_size=True)
l1 = data_structures.list_append(l, 1)
l2 = data_structures.list_append(l1, 2)
with self.cached_session() as sess:
self.assertAllEqual(self.evaluate(l1.stack()), [1])
self.assertAllEqual(self.evaluate(l2.stack()), [1, 2])
def test_append_python(self):
l = []
self.assertAllEqual(data_structures.list_append(l, 1), [1])
self.assertAllEqual(data_structures.list_append(l, 2), [1, 2])
def test_pop_tensor_list(self):
initial_list = constant_op.constant([[1, 2], [3, 4]])
elem_shape = constant_op.constant([2])
l = list_ops.tensor_list_from_tensor(initial_list, element_shape=elem_shape)
opts = data_structures.ListPopOpts(
element_dtype=initial_list.dtype,
element_shape=(2,))
with self.assertRaises(NotImplementedError):
data_structures.list_pop(l, 0, opts)
with self.cached_session() as sess:
l, x = data_structures.list_pop(l, None, opts)
self.assertAllEqual(self.evaluate(x), [3, 4])
t = list_ops.tensor_list_stack(l, element_dtype=initial_list.dtype)
self.assertAllEqual(self.evaluate(t), [[1, 2]])
def test_pop_python(self):
l = [1, 2, 3]
opts = data_structures.ListPopOpts(element_dtype=None, element_shape=())
self.assertAllEqual(data_structures.list_pop(l, None, opts), ([1, 2], 3))
self.assertAllEqual(data_structures.list_pop(l, None, opts), ([1], 2))
def test_stack_tensor_list(self):
initial_list = constant_op.constant([[1, 2], [3, 4]])
elem_shape = constant_op.constant([2])
l = list_ops.tensor_list_from_tensor(initial_list, element_shape=elem_shape)
opts = data_structures.ListStackOpts(
element_dtype=initial_list.dtype, original_call=None)
with self.cached_session() as sess:
t = data_structures.list_stack(l, opts)
self.assertAllEqual(self.evaluate(t), self.evaluate(initial_list))
@test_util.run_deprecated_v1
def test_stack_tensor_list_empty(self):
l = list_ops.empty_tensor_list(
element_shape=None, element_dtype=dtypes.variant)
opts = data_structures.ListStackOpts(
element_dtype=dtypes.int32, original_call=None)
# TODO(mdan): Allow stacking empty lists if the dtype and shape are known.
with self.assertRaises(ValueError):
data_structures.list_stack(l, opts)
def test_stack_fallback(self):
def dummy_function(l):
# Lazy person's mock: just transform the argument in a way in which we
# can check that this function was indeed called.
return [x * 2 for x in l]
opts = data_structures.ListStackOpts(
element_dtype=None, original_call=dummy_function)
self.assertAllEqual(data_structures.list_stack([1, 2], opts), [2, 4])
if __name__ == '__main__':
test.main()
tensorflow/python/autograph/operators/dispatch_context.py
+37
View File
@@ -0,0 +1,37 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Structures that allow uniform control over the dispatch process."""
import collections
# TODO(mdan): This is where macro override controls fit.
class DispatchContext(collections.namedtuple(
'DispatchContext',
('options',))):
"""Allows passing additional parameters to the specific implementations.
Attributes:
options: Optional dict of extra arguments that may be required by specific
implementations.
"""
def option(self, name):
return self.options[name]
NO_CTX = DispatchContext(options={})
tensorflow/python/autograph/operators/exceptions.py
+82
View File
@@ -0,0 +1,82 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Exception handling statements: assert, etc."""
from tensorflow.python.framework import tensor_util
from tensorflow.python.ops import control_flow_assert
from tensorflow.python.util import tf_inspect
def assert_stmt(expression1, expression2):
"""Functional form of an assert statement.
This follows the semantics of the Python assert statement, however the
concrete implementations may deviate from it. See the respective
implementation for details.
In general, the assert statement should not be used for control flow.
Furthermore, it is encouraged that the assertion expressions should not have
side effects.
Args:
expression1: Any
expression2: Callable[[], Any], returns the expression to include in the
error message when expression1 evaluates to False. When expression1 is
True, the result of expression2 will not be evaluated, however,
expression2 itself may be evaluated in some implementations.
Returns:
Any, implementation-dependent.
Raises:
ValueError: if any arguments are illegal.
"""
if not callable(expression2):
raise ValueError('{} must be a callable'.format(expression2))
args, _, keywords, _ = tf_inspect.getargspec(expression2)
if args or keywords:
raise ValueError('{} may not have any arguments'.format(expression2))
if tensor_util.is_tf_type(expression1):
return _tf_assert_stmt(expression1, expression2)
else:
return _py_assert_stmt(expression1, expression2)
def _tf_assert_stmt(expression1, expression2):
"""Overload of assert_stmt that stages a TF Assert.
This implementation deviates from Python semantics as follows:
(1) the assertion is verified regardless of the state of __debug__
(2) on assertion failure, the graph execution will fail with
tensorflow.errors.ValueError, rather than AssertionError.
Args:
expression1: tensorflow.Tensor, must evaluate to a tf.bool scalar
expression2: Callable[[], Union[tensorflow.Tensor, List[tensorflow.Tensor]]]
Returns:
tensorflow.Operation
"""
expression2_tensors = expression2()
if not isinstance(expression2_tensors, list):
expression2_tensors = [expression2_tensors]
return control_flow_assert.Assert(expression1, expression2_tensors)
def _py_assert_stmt(expression1, expression2):
"""Overload of assert_stmt that executes a Python assert statement."""
assert expression1, expression2()
return None
tensorflow/python/autograph/operators/exceptions_test.py
+86
View File
@@ -0,0 +1,86 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for exceptions module."""
from tensorflow.python.autograph.operators import exceptions
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import test_util
from tensorflow.python.platform import test
class ExceptionsTest(test.TestCase):
def test_assert_tf_untriggered(self):
with self.cached_session() as sess:
t = exceptions.assert_stmt(
constant_op.constant(True), lambda: constant_op.constant('ignored'))
self.evaluate(t)
@test_util.run_deprecated_v1
def test_assert_tf_triggered(self):
with self.cached_session() as sess:
t = exceptions.assert_stmt(
constant_op.constant(False),
lambda: constant_op.constant('test message'))
with self.assertRaisesRegex(errors_impl.InvalidArgumentError,
'test message'):
self.evaluate(t)
@test_util.run_deprecated_v1
def test_assert_tf_multiple_printed_values(self):
two_tensors = [
constant_op.constant('test message'),
constant_op.constant('another message')
]
with self.cached_session() as sess:
t = exceptions.assert_stmt(
constant_op.constant(False), lambda: two_tensors)
with self.assertRaisesRegex(errors_impl.InvalidArgumentError,
'test message.*another message'):
self.evaluate(t)
def test_assert_python_untriggered(self):
side_effect_trace = []
def expression_with_side_effects():
side_effect_trace.append(object())
return 'test message'
exceptions.assert_stmt(True, expression_with_side_effects)
self.assertListEqual(side_effect_trace, [])
def test_assert_python_triggered(self):
if not __debug__:
# Python assertions only be tested when in debug mode.
return
side_effect_trace = []
tracer = object()
def expression_with_side_effects():
side_effect_trace.append(tracer)
return 'test message'
with self.assertRaisesRegex(AssertionError, 'test message'):
exceptions.assert_stmt(False, expression_with_side_effects)
self.assertListEqual(side_effect_trace, [tracer])
if __name__ == '__main__':
test.main()
tensorflow/python/autograph/operators/logical.py
+96
View File
@@ -0,0 +1,96 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Logical boolean operators: not, and, or."""
from tensorflow.python.framework import tensor_util
from tensorflow.python.ops import cond as tf_cond
from tensorflow.python.ops import gen_math_ops
def not_(a):
"""Functional form of "not"."""
if tensor_util.is_tf_type(a):
return _tf_not(a)
return _py_not(a)
def _tf_not(a):
"""Implementation of the "not_" operator for TensorFlow."""
return gen_math_ops.logical_not(a)
def _py_not(a):
"""Default Python implementation of the "not_" operator."""
return not a
def and_(a, b):
"""Functional form of "and". Uses lazy evaluation semantics."""
a_val = a()
if tensor_util.is_tf_type(a_val):
return _tf_lazy_and(a_val, b)
return _py_lazy_and(a_val, b)
def _tf_lazy_and(cond, b):
"""Lazy-eval equivalent of "and" for Tensors."""
# TODO(mdan): Enforce cond is scalar here?
return tf_cond.cond(cond, b, lambda: cond)
def _py_lazy_and(cond, b):
"""Lazy-eval equivalent of "and" in Python."""
return cond and b()
def or_(a, b):
"""Functional form of "or". Uses lazy evaluation semantics."""
a_val = a()
if tensor_util.is_tf_type(a_val):
return _tf_lazy_or(a_val, b)
return _py_lazy_or(a_val, b)
def _tf_lazy_or(cond, b):
"""Lazy-eval equivalent of "or" for Tensors."""
# TODO(mdan): Enforce cond is scalar here?
return tf_cond.cond(cond, lambda: cond, b)
def _py_lazy_or(cond, b):
"""Lazy-eval equivalent of "or" in Python."""
return cond or b()
def eq(a, b):
"""Functional form of "equal"."""
if tensor_util.is_tf_type(a) or tensor_util.is_tf_type(b):
return _tf_equal(a, b)
return _py_equal(a, b)
def _tf_equal(a, b):
"""Overload of "equal" for Tensors."""
return gen_math_ops.equal(a, b)
def _py_equal(a, b):
"""Overload of "equal" that falls back to Python's default implementation."""
return a == b
def not_eq(a, b):
"""Functional form of "not-equal"."""
return not_(eq(a, b))
tensorflow/python/autograph/operators/logical_test.py
+84
View File
@@ -0,0 +1,84 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for logical module."""
from tensorflow.python.autograph.operators import logical
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import test_util
from tensorflow.python.platform import test
class LogicalOperatorsTest(test.TestCase):
def assertNotCalled(self):
self.fail('this should not be called')
def _tf_true(self):
return constant_op.constant(True)
def _tf_false(self):
return constant_op.constant(False)
def test_and_python(self):
self.assertTrue(logical.and_(lambda: True, lambda: True))
self.assertTrue(logical.and_(lambda: [1], lambda: True))
self.assertListEqual(logical.and_(lambda: True, lambda: [1]), [1])
self.assertFalse(logical.and_(lambda: False, lambda: True))
self.assertFalse(logical.and_(lambda: False, self.assertNotCalled))
@test_util.run_deprecated_v1
def test_and_tf(self):
with self.cached_session() as sess:
t = logical.and_(self._tf_true, self._tf_true)
self.assertEqual(self.evaluate(t), True)
t = logical.and_(self._tf_true, lambda: True)
self.assertEqual(self.evaluate(t), True)
t = logical.and_(self._tf_false, lambda: True)
self.assertEqual(self.evaluate(t), False)
# TODO(mdan): Add a test for ops with side effects.
def test_or_python(self):
self.assertFalse(logical.or_(lambda: False, lambda: False))
self.assertFalse(logical.or_(lambda: [], lambda: False))
self.assertListEqual(logical.or_(lambda: False, lambda: [1]), [1])
self.assertTrue(logical.or_(lambda: False, lambda: True))
self.assertTrue(logical.or_(lambda: True, self.assertNotCalled))
@test_util.run_deprecated_v1
def test_or_tf(self):
with self.cached_session() as sess:
t = logical.or_(self._tf_false, self._tf_true)
self.assertEqual(self.evaluate(t), True)
t = logical.or_(self._tf_false, lambda: True)
self.assertEqual(self.evaluate(t), True)
t = logical.or_(self._tf_true, lambda: True)
self.assertEqual(self.evaluate(t), True)
# TODO(mdan): Add a test for ops with side effects.
def test_not_python(self):
self.assertFalse(logical.not_(True))
self.assertFalse(logical.not_([1]))
self.assertTrue(logical.not_([]))
def test_not_tf(self):
with self.cached_session() as sess:
t = logical.not_(self._tf_false())
self.assertEqual(self.evaluate(t), True)
if __name__ == '__main__':
test.main()
tensorflow/python/autograph/operators/py_builtins.py
+533
View File
@@ -0,0 +1,533 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Operators corresponding to Python builtin functions.
List of built-in functions: https://docs.python.org/3/library/functions.html
"""
import inspect
from tensorflow.python.autograph.utils import tensors
from tensorflow.python.autograph.utils import type_registry
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import cond
from tensorflow.python.ops import control_flow_assert
from tensorflow.python.ops import gen_parsing_ops
from tensorflow.python.ops import gen_string_ops
from tensorflow.python.ops import list_ops
from tensorflow.python.ops import math_ops
UNSPECIFIED = object()
abs_registry = type_registry.TypeRegistry()
len_registry = type_registry.TypeRegistry()
print_registry = type_registry.TypeRegistry()
enumerate_registry = type_registry.TypeRegistry()
zip_registry = type_registry.TypeRegistry()
map_registry = type_registry.TypeRegistry()
filter_registry = type_registry.TypeRegistry()
any_registry = type_registry.TypeRegistry()
all_registry = type_registry.TypeRegistry()
sorted_registry = type_registry.TypeRegistry()
next_registry = type_registry.TypeRegistry()
def registry_lookup(reg, obj):
try:
return reg.lookup(obj)
except LookupError:
pass
return None
def overload_of(f):
if f in SUPPORTED_BUILTINS:
return BUILTIN_FUNCTIONS_MAP[f.__name__]
return f
def _find_originating_frame(caller_fn_scope, innermost=True):
"""Locates the frame in which `caller_fn_scope` was defined."""
ctx_frame = inspect.currentframe()
result = None
while ctx_frame is not None:
# Note it should not be normally possible to get false positives this way
# because the function scope object is not accessible to user code (barring
# call stack introspection).
if ctx_frame.f_locals.get(caller_fn_scope.name, None) is caller_fn_scope:
result = ctx_frame
if innermost:
break
ctx_frame = ctx_frame.f_back
assert result is not None, (
'the conversion process should ensure the caller_fn_scope is always'
' found somewhere on the call stack')
return result
def locals_in_original_context(caller_fn_scope):
"""Executes the locals function in the context of a specified function."""
return _find_originating_frame(caller_fn_scope, innermost=True).f_locals
def globals_in_original_context(caller_fn_scope):
"""Executes the locals function in the context of a specified function."""
return _find_originating_frame(caller_fn_scope, innermost=True).f_globals
def eval_in_original_context(f, args, caller_fn_scope):
"""Executes the eval function in the context of a specified function."""
# When control flow is rewritten using functions, eval should use the
# variables found in the same block where it was called. That is equivalent
# to the innermost function call.
ctx_frame = _find_originating_frame(caller_fn_scope, innermost=True)
args = (
args[0],
ctx_frame.f_globals if len(args) < 2 else args[1],
ctx_frame.f_locals if len(args) < 3 else args[2],
)
return f(*args)
def super_in_original_context(f, args, caller_fn_scope):
"""Executes the super function in the context of a specified function.
See https://docs.python.org/3/library/functions.html#super for the exact
details
Args:
f: Callable, typically the super builtin
args: List[Any], the original call arguments
caller_fn_scope: Optional[function_wrappers.FunctionScope], the function
scope of the converted function in which this call was originally made
Returns:
The result of calling `f` as if it was called in the frame indicated by
`caller_fn_scope`.
"""
# Only the no-arg call is desugared.
if args:
return f(*args)
# Inner functions seem to include their closure in f_locals, so we need
# to find the outermost frame.
ctx_frame = _find_originating_frame(caller_fn_scope, innermost=False)
# When super(..) is called without arguments, it looks for __class__ cell
# variable and the first argument passed in the enclosing function according
# to the spec https://www.python.org/dev/peps/pep-3135/ .
#
# We couldn't verify if `inspect.currentframe().f_code.co_varnames[0]` is
# guaranteed to be the first argument from an official doc or PEP, however,
# it's fairly stable and well established:
# - An unofficial community doc mentions it.
# https://python-reference.readthedocs.io/en/latest/docs/code/varnames.html
# - CPython has tests checking that order, which was merged in 2008, and
# unchanged since then.
# https://github.com/python/cpython/blame/2f224a077a83ac9de8a12bb7dcc516642b8176d8/Lib/lib2to3/tests/data/py2_test_grammar.py#L157
# https://github.com/python/cpython/blame/2f224a077a83ac9de8a12bb7dcc516642b8176d8/Lib/lib2to3/tests/data/py3_test_grammar.py#L192
#
# Note: the name can be more reliably obtained by inspecting the calling
# function's argspec.
#
# Even though methods can be declared using *args (def method(*args)),
# that pattern is disallowed by super() -- it raises super() no arguments.
# Method definitions using **kwargs are not allowed at all.
# In other words, we can always assume that self is on the first positional
# argument (for correct code).
#
# TODO(mdan): Consider additional checks in case the input code is incorrect.
# For example, the error might be cryptic compared to what super() regularly
# raises.
type_arg = ctx_frame.f_locals['__class__']
self_arg_name = ctx_frame.f_code.co_varnames[0]
self_arg = ctx_frame.f_locals[self_arg_name]
return f(type_arg, self_arg)
def abs_(x):
abs_override = registry_lookup(abs_registry, x)
if abs_override is not None:
return abs_override(x)
if tensor_util.is_tf_type(x):
return _tf_abs(x)
return _py_abs(x)
def _tf_abs(x):
return math_ops.abs(x)
def _py_abs(x):
return abs(x)
def float_(x=0):
if tensor_util.is_tf_type(x):
return _tf_float(x)
return _py_float(x)
def _tf_float(x):
# TODO(mdan): We shouldn't assume float32.
if x.dtype == dtypes.string:
return gen_parsing_ops.string_to_number(x, out_type=dtypes.float32)
return math_ops.cast(x, dtype=dtypes.float32)
def _py_float(x):
return float(x)
def int_(x=0, base=UNSPECIFIED):
if tensor_util.is_tf_type(x):
return _tf_int(x, base)
return _py_int(x, base)
def _tf_int(x, base):
if base not in (10, UNSPECIFIED):
raise NotImplementedError('base {} not supported for int'.format(base))
# TODO(mdan): We shouldn't assume int32.
if x.dtype == dtypes.string:
return gen_parsing_ops.string_to_number(x, out_type=dtypes.int32)
return math_ops.cast(x, dtype=dtypes.int32)
def _py_int(x, base):
if base is UNSPECIFIED:
return int(x)
return int(x, base)
def len_(s):
len_override = registry_lookup(len_registry, s)
if len_override is not None:
return len_override(s)
if tensors.is_tensor_array(s):
return _tf_tensor_array_len(s)
elif tensors.is_tensor_list(s):
return _tf_tensor_list_len(s)
elif tensor_util.is_tf_type(s):
return _tf_tensor_len(s)
return _py_len(s)
def _tf_tensor_array_len(s):
return s.size()
def _tf_tensor_list_len(s):
return list_ops.tensor_list_length(s)
def _tf_tensor_len(s):
"""Overload of len_ for Tensor arguments."""
# Statically shaped tensors: length is known ahead of time.
if s.shape.ndims and s.shape.dims[0].value is not None:
return s.shape.dims[0].value
# Static shape of unknown dimensions: use dynamic shape but statically
# check that it's a scalar.
shape = array_ops.shape(s)
assert shape.shape, 'shape tensor of zero size? {}'.format(shape)
if shape.shape[0] == 0:
raise ValueError(
'len requires a non-scalar tensor, got one of shape {}'.format(shape))
if shape.shape.dims[0].value is not None:
return array_ops.shape(s)[0]
# Fully dynamic shape: use ops.
rank = array_ops.rank(s)
def raise_zero_rank_error():
msg = gen_string_ops.string_join(
['len requires non-zero rank, got ',
gen_string_ops.as_string(rank)])
with ops.control_dependencies([control_flow_assert.Assert(False, [msg])]):
return constant_op.constant(0, dtype=dtypes.int32)
return cond.cond(rank > 0, lambda: array_ops.shape(s)[0],
raise_zero_rank_error)
def _py_len(s):
return len(s)
def print_(*objects, **kwargs):
"""Overload of the print builtin."""
# Note: Python 2.6 doesn't support explicit keywords after starargs.
unknown_kwargs = tuple(
set(kwargs.keys()) - set(('sep', 'end', 'file', 'flush')))
if unknown_kwargs:
raise ValueError('invalid keyword arguments: {}'.format(unknown_kwargs))
print_fn = _py_print
for x in objects:
print_override = registry_lookup(print_registry, x)
if print_override is not None: # pylint: disable=comparison-with-callable
print_fn = print_override
break
if print_fn is _py_print:
# If this fails, ops/autograph_ops.py hasn't been imported.
assert not any(tensor_util.is_tf_type(s) for s in objects)
return print_fn(*objects, **kwargs)
def _py_print(*objects, **kwargs):
print(*objects, **kwargs)
def min_(*args, **kwargs):
if any(tensor_util.is_tf_type(s) for s in args):
return _tf_min(*args, **kwargs)
return _py_min(*args, **kwargs)
def _tf_min(*args, **kwargs):
if len(kwargs):
kwargs_tuple = tuple(set(kwargs.keys()))
raise ValueError('These keyword arguments are '
'currently not supported: {}'.format(kwargs_tuple))
if len(args) == 1:
rank = args[0].shape.rank
if rank == 0:
return args[0]
if rank == 1:
return math_ops.reduce_min(*args, axis=0)
raise ValueError('min(arg) currently support only tensor with rank 1, '
'but got a tensor with rank {}'.format(rank))
for arg in args:
rank = arg.shape.rank
if rank != 0:
raise ValueError('min(arg1, arg2, *args) currently support '
'only scalar tensor, but got a tensor '
'with shape {}'.format(rank))
return math_ops.reduce_min(args, axis=0)
def _py_min(*args, **kwargs):
return min(*args, **kwargs)
def max_(*args, **kwargs):
if any(tensor_util.is_tf_type(s) for s in args):
return _tf_max(*args, **kwargs)
return _py_max(*args, **kwargs)
def _tf_max(*args, **kwargs):
if len(kwargs):
kwargs_tuple = tuple(set(kwargs.keys()))
raise ValueError('These keyword arguments are '
'currently not supported: {}'.format(kwargs_tuple))
if len(args) == 1:
rank = args[0].shape.rank
if rank == 0:
return args[0]
if rank == 1:
return math_ops.reduce_max(*args, axis=0)
raise ValueError('max(arg) currently support only tensor with rank 1, '
'but got a tensor with rank {}'.format(rank))
for arg in args:
rank = arg.shape.rank
if rank != 0:
raise ValueError('max(arg1, arg2, *args) currently support '
'only scalar tensor, but got a tensor '
'with shape {}'.format(rank))
return math_ops.reduce_max(args, axis=0)
def _py_max(*args, **kwargs):
return max(*args, **kwargs)
def range_(start_or_stop, stop=UNSPECIFIED, step=UNSPECIFIED):
if any(tensor_util.is_tf_type(s) for s in (start_or_stop, stop, step)):
return _tf_range(start_or_stop, stop, step)
return _py_range(start_or_stop, stop, step)
def _tf_range(start_or_stop, stop, step):
"""Overload of range_ that generates a TF range tensor."""
# Note: for static inputs (e.g. constants), tf.range errors out at graph
# construction time, instead of returning an empty tensor. Preventing the
# graph construction error aligns the semantics with Python.
# TODO(mdan): We should optimize this when a full tensor is not required.
if step is not UNSPECIFIED:
# TODO(mdan): Add argument coercion similar to other cases.
return math_ops.range(start_or_stop, stop, step)
if stop is not UNSPECIFIED:
stop = math_ops.maximum(start_or_stop, stop)
return math_ops.range(start_or_stop, stop)
start_or_stop = math_ops.maximum(start_or_stop, 0)
return math_ops.range(start_or_stop)
def _py_range(start_or_stop, stop, step):
if step is not UNSPECIFIED:
return range(start_or_stop, stop, step)
if stop is not UNSPECIFIED:
return range(start_or_stop, stop)
return range(start_or_stop)
def enumerate_(s, start=0):
enumerate_override = registry_lookup(enumerate_registry, s)
if enumerate_override is not None:
return enumerate_override(s, start)
return _py_enumerate(s, start)
def _py_enumerate(s, start=0):
return enumerate(s, start)
def zip_(*iterables, strict=False):
zip_fn = _py_zip
# If the overridden function is not the same across all iterables, use _py_zip
for x in iterables:
zip_override = registry_lookup(zip_registry, x)
if zip_override is None or (zip_fn != _py_zip and zip_override != zip_fn): # pylint: disable=comparison-with-callable
zip_fn = _py_zip
break
zip_fn = zip_override
return zip_fn(*iterables, strict=strict)
def _py_zip(*iterables, strict=False):
if strict:
return zip(*iterables, strict=True)
else:
# Python < 3.10 doesn't have `strict` kwarg.
return zip(*iterables)
def map_(fn, *iterables):
map_fn = _py_map
# If the overridden function is not the same across all iterables, use _py_map
for x in iterables:
map_override = registry_lookup(map_registry, x)
if map_override is None or (map_fn != _py_map and map_override != map_fn): # pylint: disable=comparison-with-callable
map_fn = _py_map
break
map_fn = map_override
return map_fn(fn, *iterables)
def _py_map(fn, *iterables):
return map(fn, *iterables)
def next_(iterator, default=UNSPECIFIED):
next_override = registry_lookup(next_registry, iterator)
if next_override is not None:
return next_override(iterator, default)
return next_py(iterator, default)
def next_py(iterator, default=UNSPECIFIED):
if default is UNSPECIFIED:
return next(iterator)
return next(iterator, default)
def filter_(function, iterable):
filter_override = registry_lookup(filter_registry, iterable)
if filter_override is not None:
return filter_override(function, iterable)
return _py_filter(function, iterable)
def _py_filter(function, iterable):
return filter(function, iterable)
def any_(iterable):
any_override = registry_lookup(any_registry, iterable)
if any_override is not None:
return any_override(iterable)
return _py_any(iterable)
def _py_any(iterable):
return any(iterable)
def all_(iterable):
all_override = registry_lookup(all_registry, iterable)
if all_override is not None:
return all_override(iterable)
return _py_all(iterable)
def _py_all(iterable):
return all(iterable)
def sorted_(iterable, key=UNSPECIFIED, reverse=UNSPECIFIED):
sorted_override = registry_lookup(sorted_registry, iterable)
if sorted_override is not None:
return sorted_override(iterable, key, reverse)
return _py_sorted(iterable, key, reverse)
def _py_sorted(iterable, key, reverse):
if key is not UNSPECIFIED and reverse is UNSPECIFIED:
return sorted(iterable, key=key)
if key is UNSPECIFIED and reverse is not UNSPECIFIED:
return sorted(iterable, reverse=reverse)
if key is not UNSPECIFIED and reverse is not UNSPECIFIED:
return sorted(iterable, key=key, reverse=reverse)
return sorted(iterable)
SUPPORTED_BUILTINS = (abs, float, int, len, print, range, enumerate, zip, map,
filter, any, all, sorted)
BUILTIN_FUNCTIONS_MAP = {
'abs': abs_,
'any': any_,
'all': all_,
'enumerate': enumerate_,
'filter': filter_,
'float': float_,
'int': int_,
'len': len_,
'map': map_,
'next': next_,
'print': print_,
'range': range_,
'sorted': sorted_,
'zip': zip_,
}
tensorflow/python/autograph/operators/py_builtins_test.py
+718
View File
@@ -0,0 +1,718 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for py_builtins module."""
import io
import sys
from tensorflow.python.autograph.core import converter
from tensorflow.python.autograph.core import function_wrappers
from tensorflow.python.autograph.operators import data_structures
from tensorflow.python.autograph.operators import py_builtins
from tensorflow.python.data.ops import dataset_ops
from tensorflow.python.eager import def_function
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import tensor_array_ops
from tensorflow.python.platform import test
class TestBase:
def overridden_method(self, x):
return x + 20
@test_util.run_all_in_graph_and_eager_modes
class PyBuiltinsTest(test.TestCase):
def test_abs(self):
self.assertEqual(py_builtins.abs_(-1), 1)
with self.cached_session() as sess:
t = py_builtins.abs_(constant_op.constant(-1))
self.assertEqual(self.evaluate(t), 1)
t = py_builtins.abs_(constant_op.constant([-1, 2, -3]))
self.assertAllEqual(self.evaluate(t), [1, 2, 3])
def test_abs_dataset(self):
dataset = dataset_ops.DatasetV2.from_tensor_slices([-1, 2, 3])
dataset = py_builtins.abs_(dataset)
iterator = dataset_ops.make_one_shot_iterator(dataset)
with self.cached_session() as sess:
self.assertAllEqual(self.evaluate(iterator.get_next()), 1)
self.assertAllEqual(self.evaluate(iterator.get_next()), 2)
self.assertAllEqual(self.evaluate(iterator.get_next()), 3)
def test_abs_dataset_zipped(self):
dataset_1 = dataset_ops.DatasetV2.from_tensor_slices([-1, 2, 3])
dataset_2 = dataset_ops.DatasetV2.from_tensor_slices([1, -2, 3])
dataset = dataset_ops.DatasetV2.zip((dataset_1, dataset_2))
dataset = py_builtins.abs_(dataset)
iterator = dataset_ops.make_one_shot_iterator(dataset)
with self.cached_session() as sess:
self.assertAllEqual(self.evaluate(iterator.get_next()), (1, 1))
self.assertAllEqual(self.evaluate(iterator.get_next()), (2, 2))
self.assertAllEqual(self.evaluate(iterator.get_next()), (3, 3))
def test_abs_dataset_mixed(self):
dataset_1 = dataset_ops.DatasetV2.from_tensor_slices([-1, 2, 3])
dataset_2 = dataset_ops.DatasetV2.from_tensor_slices([1, -2, 3])
dataset_3 = dataset_ops.DatasetV2.from_tensor_slices([-1, -2, -3])
dataset_4 = dataset_ops.DatasetV2.zip((dataset_1, dataset_2))
dataset = dataset_ops.DatasetV2.zip((dataset_3, dataset_4))
dataset = py_builtins.abs_(dataset)
iterator = dataset_ops.make_one_shot_iterator(dataset)
with self.cached_session() as sess:
for i in range(1, 4):
actual = self.evaluate(iterator.get_next())
self.assertAllEqual(actual[0], i)
self.assertAllEqual(actual[1], (i, i))
def test_float(self):
self.assertEqual(py_builtins.float_(10), 10.0)
self.assertEqual(py_builtins.float_('10.0'), 10.0)
with self.cached_session() as sess:
t = py_builtins.float_(constant_op.constant(1, dtype=dtypes.int64))
self.assertEqual(self.evaluate(t), 1.0)
st = py_builtins.float_(constant_op.constant('1.0'))
self.assertEqual(self.evaluate(st), 1.0)
def test_int(self):
self.assertEqual(py_builtins.int_(10.0), 10)
self.assertEqual(py_builtins.int_('11', 2), 3)
with self.cached_session() as sess:
t = py_builtins.int_(constant_op.constant(1, dtype=dtypes.float64))
self.assertEqual(self.evaluate(t), 1)
st = py_builtins.int_(constant_op.constant('1'))
self.assertEqual(self.evaluate(st), 1)
st = py_builtins.int_(constant_op.constant('1'), 10)
self.assertEqual(self.evaluate(st), 1)
def test_int_unsupported_base(self):
t = constant_op.constant(1, dtype=dtypes.float64)
with self.assertRaises(NotImplementedError):
py_builtins.int_(t, 2)
def test_len(self):
self.assertEqual(py_builtins.len_([1, 2, 3]), 3)
with self.cached_session() as sess:
t = py_builtins.len_(constant_op.constant([[1], [2], [3]]))
self.assertEqual(t, 3)
ta = py_builtins.len_(tensor_array_ops.TensorArray(dtypes.int32, size=5))
self.assertEqual(self.evaluate(ta), 5)
tl = py_builtins.len_(data_structures.tf_tensor_list_new([3, 4, 5]))
self.assertEqual(self.evaluate(tl), 3)
def test_len_dataset(self):
dataset = dataset_ops.DatasetV2.from_tensor_slices([3, 2, 1])
self.assertEqual(self.evaluate(py_builtins.len_(dataset)), 3)
# graph mode
@def_function.function(autograph=False)
def test_fn():
dataset = dataset_ops.DatasetV2.from_tensor_slices([3, 2, 1])
return py_builtins.len_(dataset)
self.assertEqual(self.evaluate(test_fn()), 3)
def test_len_dataset_infinite(self):
dataset = dataset_ops.DatasetV2.range(5).repeat().batch(2)
with self.assertRaises(errors_impl.InvalidArgumentError):
_ = self.evaluate(py_builtins.len_(dataset))
# graph mode
@def_function.function
def test_fn():
dataset = dataset_ops.DatasetV2.range(5).repeat().batch(2)
return py_builtins.len_(dataset)
with self.assertRaises(errors_impl.InvalidArgumentError):
self.evaluate(test_fn())
def test_len_dataset_unknown(self):
dataset = dataset_ops.DatasetV2.range(5).filter(lambda _: True).batch(2)
with self.assertRaises(errors_impl.InvalidArgumentError):
_ = self.evaluate(py_builtins.len_(dataset))
# graph mode
@def_function.function(autograph=False)
def test_fn():
dataset = dataset_ops.DatasetV2.range(5).filter(lambda _: True).batch(2)
return py_builtins.len_(dataset)
with self.assertRaises(errors_impl.InvalidArgumentError):
self.evaluate(test_fn())
def test_len_scalar(self):
with self.assertRaises(ValueError):
py_builtins.len_(constant_op.constant(1))
@test_util.run_deprecated_v1
def test_len_dynamic_shape(self):
with self.cached_session() as sess:
p = array_ops.placeholder(dtype=dtypes.int32, shape=None)
t = py_builtins.len_(p)
self.assertEqual(sess.run(t, {p: [1, 2, 3]}), 3)
with self.assertRaises(errors_impl.InvalidArgumentError):
t = py_builtins.len_(p)
sess.run(t, {p: 1})
@test_util.run_deprecated_v1
def test_print_tensors(self):
try:
out_capturer = io.StringIO()
sys.stdout = out_capturer
with self.cached_session() as sess:
sess.run(py_builtins.print_(constant_op.constant('test message'), 1))
self.assertEqual(out_capturer.getvalue(), 'test message 1\n')
finally:
sys.stdout = sys.__stdout__
@test_util.run_deprecated_v1
def test_print_complex(self):
try:
out_capturer = io.StringIO()
sys.stdout = out_capturer
with self.cached_session() as sess:
sess.run(
py_builtins.print_(constant_op.constant('test message'), [1, 2]))
self.assertEqual(out_capturer.getvalue(), 'test message [1, 2]\n')
finally:
sys.stdout = sys.__stdout__
def test_max(self):
self.assertEqual(py_builtins.max_([1, 3, 2]), 3)
self.assertEqual(py_builtins.max_(0, 2, 1), 2)
def test_max_tensor(self):
r = py_builtins.max_(constant_op.constant(2))
self.assertAllEqual(self.evaluate(r), 2)
with self.assertRaises(ValueError):
py_builtins.max_(constant_op.constant([[2]]))
r = py_builtins.max_(constant_op.constant([1, 3, 2]))
self.assertAllEqual(self.evaluate(r), 3)
with self.assertRaises(ValueError):
py_builtins.max_(constant_op.constant([[1, 3], [3, 4]]))
r = py_builtins.max_(
constant_op.constant(6), constant_op.constant(4),
constant_op.constant(8))
self.assertAllEqual(self.evaluate(r), 8)
with self.assertRaises(ValueError):
py_builtins.max_(
constant_op.constant([6]), constant_op.constant(4),
constant_op.constant(8))
def test_min(self):
self.assertEqual(py_builtins.min_([2, 1, 3]), 1)
self.assertEqual(py_builtins.min_(2, 0, 1), 0)
def test_min_tensor(self):
r = py_builtins.min_(constant_op.constant(2))
self.assertAllEqual(self.evaluate(r), 2)
with self.assertRaises(ValueError):
py_builtins.min_(constant_op.constant([[2]]))
r = py_builtins.min_(constant_op.constant([3, 1, 2]))
self.assertAllEqual(self.evaluate(r), 1)
with self.assertRaises(ValueError):
py_builtins.min_(constant_op.constant([[1, 3], [3, 4]]))
r = py_builtins.min_(
constant_op.constant(6), constant_op.constant(4),
constant_op.constant(8))
self.assertAllEqual(self.evaluate(r), 4)
with self.assertRaises(ValueError):
py_builtins.min_(
constant_op.constant([6]), constant_op.constant(4),
constant_op.constant(8))
def test_range(self):
self.assertListEqual(list(py_builtins.range_(3)), [0, 1, 2])
self.assertListEqual(list(py_builtins.range_(1, 3)), [1, 2])
self.assertListEqual(list(py_builtins.range_(2, 0, -1)), [2, 1])
def test_range_tensor(self):
with self.cached_session() as sess:
r = py_builtins.range_(constant_op.constant(3))
self.assertAllEqual(self.evaluate(r), [0, 1, 2])
r = py_builtins.range_(1, constant_op.constant(3))
self.assertAllEqual(self.evaluate(r), [1, 2])
r = py_builtins.range_(2, 0, constant_op.constant(-1))
self.assertAllEqual(self.evaluate(r), [2, 1])
def test_range_tensor_empty_range(self):
with self.session() as sess:
r = py_builtins.range_(constant_op.constant(-3))
self.assertAllEqual(self.evaluate(r), [])
r = py_builtins.range_(5, constant_op.constant(2))
self.assertAllEqual(self.evaluate(r), [])
def test_enumerate(self):
self.assertListEqual(
list(py_builtins.enumerate_([3, 2, 1])), [(0, 3), (1, 2), (2, 1)])
self.assertListEqual(
list(py_builtins.enumerate_([3, 2, 1], 5)), [(5, 3), (6, 2), (7, 1)])
self.assertListEqual(list(py_builtins.enumerate_([-8], -3)), [(-3, -8)])
def test_enumerate_dataset(self):
dataset = dataset_ops.DatasetV2.from_tensor_slices(['a', 'c'])
start = constant_op.constant(20, dtype=dtypes.int64)
dataset = py_builtins.enumerate_(dataset, start)
iterator = dataset_ops.make_one_shot_iterator(dataset)
with self.cached_session() as sess:
self.assertAllEqual(self.evaluate(iterator.get_next()), (20, b'a'))
self.assertAllEqual(self.evaluate(iterator.get_next()), (21, b'c'))
def test_zip(self):
self.assertListEqual(
list(py_builtins.zip_([3, 2, 1], [1, 2, 3])), [(3, 1), (2, 2), (1, 3)])
self.assertListEqual(
list(py_builtins.zip_([4, 5, 6], [-1, -2])), [(4, -1), (5, -2)])
def test_zip_dataset(self):
ds1 = dataset_ops.DatasetV2.from_tensor_slices([-11, -12, 4])
ds2 = dataset_ops.DatasetV2.from_tensor_slices([-21, -22, 5])
ds3 = py_builtins.zip_(ds1, ds2)
iterator = dataset_ops.make_one_shot_iterator(ds3)
with self.cached_session() as sess:
self.assertAllEqual(self.evaluate(iterator.get_next()), (-11, -21))
self.assertAllEqual(self.evaluate(iterator.get_next()), (-12, -22))
self.assertAllEqual(self.evaluate(iterator.get_next()), (4, 5))
def test_map(self):
def increment(x):
return x + 1
add_list = lambda x, y: x + y
self.assertListEqual(
list(py_builtins.map_(increment, [4, 5, 6])), [5, 6, 7])
self.assertListEqual(
list(py_builtins.map_(add_list, [3, 2, 1], [-1, -2, -3])), [2, 0, -2])
def test_map_dataset(self):
def increment(x):
return x + 1
ds1 = dataset_ops.DatasetV2.from_tensor_slices([4, 5, 6])
ds2 = py_builtins.map_(increment, ds1)
iterator = dataset_ops.make_one_shot_iterator(ds2)
with self.cached_session() as sess:
self.assertAllEqual(self.evaluate(iterator.get_next()), 5)
self.assertAllEqual(self.evaluate(iterator.get_next()), 6)
self.assertAllEqual(self.evaluate(iterator.get_next()), 7)
def test_map_multiple_datasets(self):
add_list = lambda x, y: x + y
ds1 = dataset_ops.DatasetV2.from_tensor_slices([-11, -12, 4])
ds2 = dataset_ops.DatasetV2.from_tensor_slices([-21, -22, 5])
ds3 = py_builtins.map_(add_list, ds1, ds2)
iterator = dataset_ops.make_one_shot_iterator(ds3)
with self.cached_session() as sess:
self.assertAllEqual(self.evaluate(iterator.get_next()), -32)
self.assertAllEqual(self.evaluate(iterator.get_next()), -34)
self.assertAllEqual(self.evaluate(iterator.get_next()), 9)
def test_next_normal(self):
iterator = iter([1, 2, 3])
self.assertEqual(py_builtins.next_(iterator), 1)
self.assertEqual(py_builtins.next_(iterator), 2)
self.assertEqual(py_builtins.next_(iterator), 3)
with self.assertRaises(StopIteration):
py_builtins.next_(iterator)
self.assertEqual(py_builtins.next_(iterator, 4), 4)
def test_next_tf_iterator(self):
# graph-mode iterators are only supported inside tf.function.
@def_function.function(autograph=False)
def test_fn(go_out_of_range, with_default):
iterator = iter(dataset_ops.Dataset.range(3))
retval = (
py_builtins.next_(iterator),
py_builtins.next_(iterator),
py_builtins.next_(iterator),
)
if go_out_of_range:
if with_default:
retval += (
py_builtins.next_(iterator,
constant_op.constant(-3, dtype=dtypes.int64)),
py_builtins.next_(iterator,
constant_op.constant(-4, dtype=dtypes.int64)),
)
else:
py_builtins.next_(iterator)
return retval
self.assertAllEqual(
self.evaluate(test_fn(go_out_of_range=False, with_default=None)),
(0, 1, 2))
self.assertAllEqual(
self.evaluate(test_fn(go_out_of_range=True, with_default=True)),
(0, 1, 2, -3, -4))
with self.assertRaises(errors_impl.OutOfRangeError):
self.evaluate(test_fn(go_out_of_range=True, with_default=False))
def test_next_tf_iterator_error_checking(self):
# graph-mode iterators are only supported inside tf.function.
@def_function.function(autograph=False)
def test_fn():
iterator = iter(dataset_ops.Dataset.range(1))
py_builtins.next_(iterator)
py_builtins.next_(iterator, constant_op.constant(-3))
# Dataset.range defaults to int64,
with self.assertRaisesRegex(TypeError, 'default.*int64'):
self.evaluate(test_fn())
def test_next_tf_iterator_error_checking_structures(self):
# graph-mode iterators are only supported inside tf.function.
@def_function.function(autograph=False)
def test_fn(default_val):
ds = dataset_ops.Dataset.range(1)
ds = ds.map(lambda i: {'a': i + 1, 'b': i + 10})
iterator = iter(ds)
py_builtins.next_(iterator)
py_builtins.next_(iterator, default_val)
default = {
'a': constant_op.constant(3, dtype=dtypes.int64),
}
with self.assertRaisesRegex(TypeError, 'same element structure'):
test_fn(default)
default = {
'a': constant_op.constant(3.0),
'b': [constant_op.constant(30), constant_op.constant(300)]
}
with self.assertRaisesRegex(TypeError, 'same element structure'):
test_fn(default)
default = {
'a': constant_op.constant(3.0),
'b': constant_op.constant(30, dtype=dtypes.int64),
}
with self.assertRaisesRegex(TypeError, 'float32'):
test_fn(default)
def _basic_function_scope(self):
return function_wrappers.FunctionScope(
'test_function_name',
'test_scope', # Note: this must match the name in the `with` statement.
converter.ConversionOptions())
def test_eval_in_original_context(self):
def test_fn():
l = 1 # pylint:disable=unused-variable
with self._basic_function_scope() as test_scope:
return py_builtins.eval_in_original_context(eval, ('l',), test_scope)
self.assertEqual(test_fn(), 1)
def test_eval_in_original_context_inner_function(self):
def test_fn():
l = 1 # pylint:disable=unused-variable
with self._basic_function_scope() as test_scope:
def inner_fn():
# Note: a user function without a top-level function scope should
# never be found in user code; it's only possible in generated code.
l = 2 # pylint:disable=unused-variable
return py_builtins.eval_in_original_context(eval, ('l',), test_scope)
return inner_fn()
self.assertEqual(test_fn(), 2)
def test_locals_in_original_context(self):
def test_fn():
l = 1 # pylint:disable=unused-variable
with self._basic_function_scope() as test_scope:
return py_builtins.locals_in_original_context(test_scope)
locs = test_fn()
self.assertEqual(locs['l'], 1)
def test_locals_in_original_context_inner_function(self):
def test_fn():
l = 1 # pylint:disable=unused-variable
with self._basic_function_scope() as test_scope:
def inner_fn():
# Note: a user function without a top-level function scope should
# never be found in user code; it's only possible in generated code.
l = 2 # pylint:disable=unused-variable
return py_builtins.locals_in_original_context(test_scope)
return inner_fn()
locs = test_fn()
self.assertEqual(locs['l'], 2)
def test_globals_in_original_context(self):
def test_fn():
with self._basic_function_scope() as test_scope:
return py_builtins.globals_in_original_context(test_scope)
globs = test_fn()
self.assertIs(globs['TestBase'], TestBase)
def test_globals_in_original_context_inner_function(self):
def test_fn():
with self._basic_function_scope() as test_scope:
def inner_fn():
# Note: a user function without a top-level function scope should
# never be found in user code; it's only possible in generated code.
return py_builtins.globals_in_original_context(test_scope)
return inner_fn()
globs = test_fn()
self.assertIs(globs['TestBase'], TestBase)
def test_super_in_original_context_unary_call(self):
test_case_self = self
class TestSubclass(TestBase):
def overridden_method(self, x):
test_case_self.fail('This should never be called.')
def test_method(self):
with test_case_self._basic_function_scope() as test_scope:
test_base_unbound = py_builtins.super_in_original_context(
super, (TestSubclass,), test_scope)
test_base = test_base_unbound.__get__(self, TestSubclass)
return test_base.overridden_method(1)
tc = TestSubclass()
self.assertEqual(tc.test_method(), 21)
def test_super_in_original_context_binary_call(self):
test_case_self = self
class TestSubclass(TestBase):
def overridden_method(self, x):
test_case_self.fail('This should never be called.')
def test_method(self):
with test_case_self._basic_function_scope() as test_scope:
test_base = py_builtins.super_in_original_context(
super, (TestSubclass, self), test_scope)
return test_base.overridden_method(1)
tc = TestSubclass()
self.assertEqual(tc.test_method(), 21)
def test_super_in_original_context_niladic_call(self):
test_case_self = self
class TestSubclass(TestBase):
def overridden_method(self, x):
test_case_self.fail('This should never be called.')
def test_method(self):
with test_case_self._basic_function_scope() as test_scope:
b = py_builtins.super_in_original_context(super, (), test_scope)
return b.overridden_method(1)
tc = TestSubclass()
self.assertEqual(tc.test_method(), 21)
def test_super_in_original_context_caller_with_locals(self):
test_case_self = self
class TestSubclass(TestBase):
def overridden_method(self, x):
test_case_self.fail('This should never be called.')
def test_method(self, x):
y = 7
with test_case_self._basic_function_scope() as test_scope:
z = 7
return py_builtins.super_in_original_context(
super, (), test_scope).overridden_method(x + y - z)
tc = TestSubclass()
self.assertEqual(tc.test_method(1), 21)
def test_super_in_original_context_inner_function(self):
test_case_self = self
class TestSubclass(TestBase):
def overridden_method(self, x):
test_case_self.fail('This should never be called.')
def test_method(self, x):
with test_case_self._basic_function_scope() as test_scope:
# Oddly, it's sufficient to use `self` in an inner function
# to gain access to __class__ in this scope.
# TODO(mdan): Is this true across implementations?
# Note: normally, it's illegal to use super() in inner functions (it
# throws an error), but the generated code may create them.
def inner_fn():
return py_builtins.super_in_original_context(
super, (), test_scope).overridden_method(x)
return inner_fn()
tc = TestSubclass()
self.assertEqual(tc.test_method(1), 21)
def test_super_in_original_context_inner_lambda(self):
test_case_self = self
class TestSubclass(TestBase):
def overridden_method(self, x):
test_case_self.fail('This should never be called.')
def test_method(self, x):
with test_case_self._basic_function_scope() as test_scope:
# Oddly, it's sufficient to use `self` in an inner function
# to gain access to __class__ in this scope.
# TODO(mdan): Is this true across implementations?
# Note: normally, it's illegal to use super() in inner functions (it
# throws an error), but the generated code may create them.
l = lambda: py_builtins.super_in_original_context( # pylint:disable=g-long-lambda
super, (), test_scope).overridden_method(x)
return l()
tc = TestSubclass()
self.assertEqual(tc.test_method(1), 21)
def test_filter(self):
self.assertListEqual(
list(py_builtins.filter_(lambda x: x == 'b', ['a', 'b', 'c'])), ['b'])
self.assertListEqual(
list(py_builtins.filter_(lambda x: x < 3, [3, 2, 1])), [2, 1])
def test_filter_dataset(self):
dataset = dataset_ops.DatasetV2.from_tensor_slices([3, 2, 1])
dataset = py_builtins.filter_(lambda x: x < 3, dataset)
iterator = dataset_ops.make_one_shot_iterator(dataset)
with self.cached_session() as sess:
self.assertAllEqual(self.evaluate(iterator.get_next()), 2)
self.assertAllEqual(self.evaluate(iterator.get_next()), 1)
def test_any(self):
self.assertEqual(py_builtins.any_([False, True, False]), True)
self.assertEqual(py_builtins.any_([False, False, False]), False)
def test_any_dataset(self):
dataset_1 = dataset_ops.DatasetV2.from_tensor_slices([False, True, False])
dataset_2 = dataset_ops.DatasetV2.from_tensor_slices([False, False, False])
self.assertEqual(self.evaluate(py_builtins.any_(dataset_1)), True)
self.assertEqual(self.evaluate(py_builtins.any_(dataset_2)), False)
dataset_3 = dataset_ops.DatasetV2.from_tensor_slices([0, 1, 2])
with self.assertRaises(ValueError):
py_builtins.any_(dataset_3)
dataset_4 = dataset_ops.DatasetV2.from_tensor_slices([False, True, False])
dataset_zipped = dataset_ops.DatasetV2.zip((dataset_4, dataset_4))
with self.assertRaises(ValueError):
py_builtins.any_(dataset_zipped)
dataset_mixed = dataset_ops.DatasetV2.zip((dataset_3, dataset_4))
with self.assertRaises(ValueError):
py_builtins.any_(dataset_mixed)
def test_all(self):
self.assertEqual(py_builtins.all_([False, True, False]), False)
self.assertEqual(py_builtins.all_([True, True, True]), True)
def test_all_dataset(self):
dataset_1 = dataset_ops.DatasetV2.from_tensor_slices([False, True, False])
dataset_2 = dataset_ops.DatasetV2.from_tensor_slices([True, True, True])
self.assertEqual(self.evaluate(py_builtins.all_(dataset_1)), False)
self.assertEqual(self.evaluate(py_builtins.all_(dataset_2)), True)
dataset_3 = dataset_ops.DatasetV2.from_tensor_slices([0, 1, 2])
with self.assertRaises(ValueError):
py_builtins.all_(dataset_3)
dataset_4 = dataset_ops.DatasetV2.from_tensor_slices([False, True, False])
dataset_zipped = dataset_ops.DatasetV2.zip((dataset_4, dataset_4))
with self.assertRaises(ValueError):
py_builtins.all_(dataset_zipped)
dataset_mixed = dataset_ops.DatasetV2.zip((dataset_3, dataset_4))
with self.assertRaises(ValueError):
py_builtins.all_(dataset_mixed)
def test_sorted(self):
self.assertListEqual(py_builtins.sorted_([2, 3, 1]), [1, 2, 3])
self.assertListEqual(
py_builtins.sorted_([2, 3, 1], key=lambda x: -x), [3, 2, 1])
self.assertListEqual(
py_builtins.sorted_([2, 3, 1], reverse=True), [3, 2, 1])
self.assertListEqual(
py_builtins.sorted_([2, 3, 1], key=lambda x: -x, reverse=True),
[1, 2, 3])
self.assertAllEqual(
py_builtins.sorted_([[4, 3], [2, 1]], key=lambda x: sum(x)),
[[2, 1], [4, 3]])
def test_sorted_tensor(self):
iterable_1 = constant_op.constant([2, 3, 1])
self.assertListEqual(
list(self.evaluate(py_builtins.sorted_(iterable_1))), [1, 2, 3])
self.assertListEqual(
list(self.evaluate(py_builtins.sorted_(iterable_1, key=lambda x: -x))),
[3, 2, 1])
self.assertListEqual(
list(self.evaluate(py_builtins.sorted_(iterable_1, reverse=True))),
[3, 2, 1])
self.assertListEqual(
list(
self.evaluate(
py_builtins.sorted_(iterable_1, key=lambda x: -x,
reverse=True))), [1, 2, 3])
iterable_2 = constant_op.constant([[4, 3], [2, 1]])
with self.assertRaises(ValueError):
py_builtins.sorted_(iterable_2)
with self.assertRaises(ValueError):
py_builtins.sorted_(iterable_2, key=lambda x: -x)
self.assertAllEqual(
list(
self.evaluate(
py_builtins.sorted_(
iterable_2, key=lambda x: math_ops.reduce_sum(x)))),
[[2, 1], [4, 3]])
if __name__ == '__main__':
test.main()
tensorflow/python/autograph/operators/slices.py
+142
View File
@@ -0,0 +1,142 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Operators specific to slicing operations."""
import collections
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import tensor_util
from tensorflow.python.ops import gen_array_ops
from tensorflow.python.ops import gen_string_ops
from tensorflow.python.ops import list_ops
from tensorflow.python.ops import tensor_array_ops
# TODO(mdan): Support extended slices.
class GetItemOpts(collections.namedtuple('GetItemOpts', ('element_dtype',))):
pass
def get_item(target, i, opts):
"""The slice read operator (i.e. __getitem__).
Note: it is unspecified whether target will be mutated or not. In general,
if target is mutable (like Python lists), it will be mutated.
Args:
target: An entity that supports getitem semantics.
i: Index to read from.
opts: A GetItemOpts object.
Returns:
The read element.
Raises:
ValueError: if target is not of a supported type.
"""
assert isinstance(opts, GetItemOpts)
if isinstance(target, tensor_array_ops.TensorArray):
return _tf_tensorarray_get_item(target, i)
elif tensor_util.is_tf_type(target):
if target.dtype == dtypes.variant:
return _tf_tensor_list_get_item(target, i, opts)
elif target.dtype == dtypes.string and target.shape.ndims == 0:
return _tf_tensor_string_get_item(target, i)
else:
return _tf_tensor_get_item(target, i)
else:
return _py_get_item(target, i)
def _tf_tensorarray_get_item(target, i):
"""Overload of get_item that stages a TensorArray read."""
return target.read(i)
def _tf_tensor_list_get_item(target, i, opts):
"""Overload of get_item that stages a Tensor list read."""
if opts.element_dtype is None:
raise ValueError('cannot retrieve from a list without knowing its '
'element type; use set_element_type to annotate it')
x = list_ops.tensor_list_get_item(target, i, element_dtype=opts.element_dtype)
return x
def _tf_tensor_get_item(target, i):
"""Overload of get_item that stages a Tensor (not Tensor list) read."""
return target[i]
def _tf_tensor_string_get_item(target, i):
"""Overload of get_item that stages a Tensor string read."""
x = gen_string_ops.substr(target, i, 1)
return x
def _py_get_item(target, i):
"""Overload of get_item that executes a Python list modification."""
return target[i]
def set_item(target, i, x):
"""The slice write operator (i.e. __setitem__).
Note: it is unspecified whether target will be mutated or not. In general,
if target is mutable (like Python lists), it will be mutated.
Args:
target: An entity that supports setitem semantics.
i: Index to modify.
x: The new element value.
Returns:
Same as target, after the update was performed.
Raises:
ValueError: if target is not of a supported type.
"""
if isinstance(target, tensor_array_ops.TensorArray):
return _tf_tensorarray_set_item(target, i, x)
elif tensor_util.is_tf_type(target):
if target.dtype == dtypes.variant:
return _tf_tensor_list_set_item(target, i, x)
else:
return _tf_tensor_set_item(target, i, x)
else:
return _py_set_item(target, i, x)
def _tf_tensorarray_set_item(target, i, x):
"""Overload of set_item that stages a TensorArray write."""
return target.write(i, x)
def _tf_tensor_list_set_item(target, i, x):
"""Overload of set_item that stages a Tensor list update."""
return list_ops.tensor_list_set_item(target, i, x)
def _tf_tensor_set_item(target, i, x):
"""Overload of set_item that stages a Tensor scatter update."""
return gen_array_ops.tensor_scatter_update(target, ((i,),), (x,))
def _py_set_item(target, i, x):
"""Overload of set_item that executes a Python list modification."""
target[i] = x
return target
tensorflow/python/autograph/operators/slices_test.py
+62
View File
@@ -0,0 +1,62 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for slices module."""
from tensorflow.python.autograph.operators import slices
from tensorflow.python.framework import constant_op
from tensorflow.python.ops import list_ops
from tensorflow.python.platform import test
class SlicesTest(test.TestCase):
def test_set_item_tensor_list(self):
initial_list = constant_op.constant([[1, 2], [3, 4]])
elem_shape = constant_op.constant([2])
l = list_ops.tensor_list_from_tensor(initial_list, element_shape=elem_shape)
l = slices.set_item(l, 0, [5, 6])
with self.cached_session() as sess:
t = list_ops.tensor_list_stack(l, element_dtype=initial_list.dtype)
self.assertAllEqual(self.evaluate(t), [[5, 6], [3, 4]])
def test_get_item_tensor_list(self):
initial_list = constant_op.constant([[1, 2], [3, 4]])
elem_shape = constant_op.constant([2])
l = list_ops.tensor_list_from_tensor(initial_list, element_shape=elem_shape)
t = slices.get_item(
l, 1, slices.GetItemOpts(element_dtype=initial_list.dtype))
with self.cached_session() as sess:
self.assertAllEqual(self.evaluate(t), [3, 4])
def test_get_item_tensor_string(self):
initial_str = constant_op.constant('abcd')
t = slices.get_item(initial_str, 1,
slices.GetItemOpts(element_dtype=initial_str.dtype))
with self.cached_session() as sess:
self.assertEqual(self.evaluate(t), b'b')
initial_list_str = constant_op.constant(['abcd', 'bcde'])
t = slices.get_item(initial_list_str, 1,
slices.GetItemOpts(element_dtype=initial_str.dtype))
with self.cached_session() as sess:
self.assertEqual(self.evaluate(t), b'bcde')
if __name__ == '__main__':
test.main()
tensorflow/python/autograph/operators/variables.py
+104
View File
@@ -0,0 +1,104 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utilities used to capture Python idioms."""
def ld(v):
"""Load variable operator."""
if isinstance(v, Undefined):
return v.read()
return v
def ldu(load_v, name):
"""Load variable operator that returns Undefined when failing to evaluate.
Note: the name ("load or return undefined") is abbreviated to minimize
the amount of clutter in generated code.
This variant of `ld` is useful when loading symbols that may be undefined at
runtime, such as composite symbols, and whether they are defined or not cannot
be determined statically. For example `d['a']` is undefined when `d` is an
empty dict.
Args:
load_v: Lambda that executes the actual read.
name: Human-readable name of the symbol being read.
Returns:
Either the value of the symbol, or Undefined, if the symbol is not fully
defined.
"""
try:
# TODO(mdan): Use locals()/globals() here.
return load_v()
except (KeyError, AttributeError, NameError):
return Undefined(name)
class Undefined(object):
"""Represents an undefined symbol in Python.
This is used to reify undefined symbols, which is required to use the
functional form of loops.
Example:
while n > 0:
n = n - 1
s = n
return s # Runtime error if n == 0
This is valid Python code and will not result in an error as long as n
is positive. The use of this class is to stay as close to Python semantics
as possible for staged code of this nature.
Converted version of the above showing the possible usage of this class:
s = Undefined('s')
init_state = (s,)
s = while_loop(cond, body, init_state)
return s # s is an instance of Undefined if the loop never runs
Attributes:
symbol_name: Text, identifier for the undefined symbol
"""
__slots__ = ('symbol_name',)
def __init__(self, symbol_name):
self.symbol_name = symbol_name
def read(self):
raise UnboundLocalError("'{}' is used before assignment".format(
self.symbol_name))
def __repr__(self):
return self.symbol_name
def __getattribute__(self, name):
try:
# If it's an existing attribute, return it.
return object.__getattribute__(self, name)
except AttributeError:
# Otherwise return Undefined.
return self
def __getitem__(self, i):
return self
# TODO(mdan): Refactor as a RetVal object, aggregating the value and do_return.
class UndefinedReturnValue(object):
"""Represents a return value that is undefined."""
pass
tensorflow/python/autograph/operators/variables_test.py
+51
View File
@@ -0,0 +1,51 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for python_lang_utils module."""
from tensorflow.python.autograph.operators import variables
from tensorflow.python.platform import test
class SpecialValuesTest(test.TestCase):
def test_undefined(self):
undefined_symbol = variables.Undefined('name')
undefined_symbol2 = variables.Undefined('name')
self.assertEqual(undefined_symbol.symbol_name, 'name')
self.assertEqual(undefined_symbol2.symbol_name, 'name')
self.assertNotEqual(undefined_symbol, undefined_symbol2)
def test_undefined_operations(self):
undefined_symbol = variables.Undefined('name')
self.assertIsInstance(undefined_symbol.foo, variables.Undefined)
self.assertIsInstance(undefined_symbol[0], variables.Undefined)
self.assertNotIsInstance(undefined_symbol.__class__, variables.Undefined)
def test_read(self):
self.assertEqual(variables.ld(1), 1)
o = object()
self.assertEqual(variables.ld(o), o)
self.assertIsNone(variables.ld(None))
def test_read_undefined(self):
with self.assertRaisesRegex(UnboundLocalError, 'used before assignment'):
variables.ld(variables.Undefined('a'))
if __name__ == '__main__':
test.main()