wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/tools/compatibility/BUILD
+289
View File
@@ -0,0 +1,289 @@
load("@xla//third_party/rules_python/python:py_binary.bzl", "py_binary")
load("@xla//third_party/rules_python/python:py_library.bzl", "py_library")
load("//tensorflow:tensorflow.bzl", "py_test")
load("//tensorflow:tensorflow.default.bzl", "get_compatible_with_portable")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = [
"//learning/brain/testing/tf2_migration_tools:__pkg__",
"//tensorflow:internal",
],
licenses = ["notice"],
)
py_library(
name = "ipynb",
srcs = ["ipynb.py"],
strict_deps = True,
)
py_library(
name = "ast_edits",
srcs = ["ast_edits.py"],
strict_deps = True,
deps = [
"@pypi//google_pasta",
"@six_archive//:six", # buildcleaner: keep
],
)
py_test(
name = "ast_edits_test",
srcs = ["ast_edits_test.py"],
strict_deps = True,
deps = [
":ast_edits",
# copybara:uncomment "//third_party/py/google/protobuf:use_fast_cpp_protos",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
"@pypi//google_pasta",
],
)
py_binary(
name = "tf_upgrade",
srcs = ["tf_upgrade.py"],
strict_deps = True,
deps = [
":tf_upgrade_lib",
"@six_archive//:six",
],
)
py_library(
name = "tf_upgrade_lib",
srcs = ["tf_upgrade.py"],
strict_deps = True,
deps = [":ast_edits"],
)
py_test(
name = "tf_upgrade_test",
srcs = ["tf_upgrade_test.py"],
strict_deps = True,
tags = [
"no_pip",
],
deps = [
":ast_edits",
":tf_upgrade_lib",
# copybara:uncomment "//third_party/py/google/protobuf:use_fast_cpp_protos",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
],
)
py_library(
name = "renames_v2",
srcs = ["renames_v2.py"],
compatible_with = get_compatible_with_portable(),
strict_deps = True,
visibility = ["//tensorflow:internal"],
)
py_library(
name = "reorders_v2",
srcs = ["reorders_v2.py"],
strict_deps = True,
)
py_library(
name = "all_renames_v2",
srcs = ["all_renames_v2.py"],
compatible_with = get_compatible_with_portable(),
strict_deps = True,
visibility = ["//tensorflow:__subpackages__"],
deps = [":renames_v2"],
)
py_test(
name = "all_renames_v2_test",
srcs = ["all_renames_v2_test.py"],
strict_deps = True,
deps = [
":all_renames_v2",
# copybara:uncomment "//third_party/py/google/protobuf:use_fast_cpp_protos",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
],
)
py_library(
name = "module_deprecations_v2",
srcs = ["module_deprecations_v2.py"],
strict_deps = True,
deps = [":ast_edits"],
)
py_library(
name = "tf_upgrade_v2_lib",
srcs = ["tf_upgrade_v2.py"],
strict_deps = True,
deps = [
":all_renames_v2",
":ast_edits",
":module_deprecations_v2",
":reorders_v2",
"@pypi//google_pasta",
],
)
py_library(
name = "tf_upgrade_v2_safety_lib",
srcs = ["tf_upgrade_v2_safety.py"],
strict_deps = True,
deps = [
":all_renames_v2",
":ast_edits",
":module_deprecations_v2",
],
)
py_binary(
name = "tf_upgrade_v2",
srcs = ["tf_upgrade_v2_main.py"],
main = "tf_upgrade_v2_main.py",
strict_deps = True,
deps = [
":ast_edits",
":ipynb",
":tf_upgrade_v2_lib",
":tf_upgrade_v2_safety_lib",
],
)
py_test(
name = "tf_upgrade_v2_test",
srcs = ["tf_upgrade_v2_test.py"],
shard_count = 5,
strict_deps = True,
tags = [
"no_windows",
"v1only",
],
deps = [
":ast_edits",
":tf_upgrade_v2_lib",
# copybara:uncomment "//third_party/py/google/protobuf:use_fast_cpp_protos",
"@six_archive//:six",
# copybara:uncomment "//third_party/py/tensorflow:tensorflow_compat_v1_estimator",
# copybara:uncomment "//third_party/py/tensorflow:tensorflow_compat_v2_estimator",
"//tensorflow:tensorflow_py",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/util:tf_decorator_py",
"//tensorflow/python/util:tf_export",
"//tensorflow/python/util:tf_inspect",
"//tensorflow/tools/common:public_api",
"//tensorflow/tools/common:traverse",
"@absl_py//absl/testing:parameterized",
],
)
py_test(
name = "tf_upgrade_v2_safety_test",
srcs = ["tf_upgrade_v2_safety_test.py"],
strict_deps = True,
deps = [
":ast_edits",
":tf_upgrade_v2_safety_lib",
# copybara:uncomment "//third_party/py/google/protobuf:use_fast_cpp_protos",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
],
)
# Keep for reference, this test will succeed in 0.11 but fail in 1.0
# py_test(
# name = "test_file_v0_11",
# size = "small",
# srcs = ["testdata/test_file_v0_11.py"],
# srcs_version = "PY3",
# deps = [
# "//tensorflow:tensorflow_py",
# ],
# )
genrule(
name = "generate_upgraded_file",
testonly = 1,
srcs = ["testdata/test_file_v0_11.py"],
outs = [
"test_file_v1_0.py",
"report.txt",
],
cmd = ("$(location :tf_upgrade)" +
" --infile $(location testdata/test_file_v0_11.py)" +
" --outfile $(location test_file_v1_0.py)" +
" --reportfile $(location report.txt)"),
tools = [":tf_upgrade"],
)
py_test(
name = "test_file_v1_0",
size = "medium",
srcs = ["test_file_v1_0.py"],
strict_deps = True,
deps = [
"//tensorflow:tensorflow_py",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
"//third_party/py/numpy",
],
)
genrule(
name = "generate_upgraded_file_v2",
testonly = 1,
srcs = ["testdata/test_file_v1_12.py"],
outs = [
"test_file_v2_0.py",
"report_v2.txt",
],
cmd = ("$(location :tf_upgrade_v2)" +
" --infile $(location testdata/test_file_v1_12.py)" +
" --outfile $(location test_file_v2_0.py)" +
" --reportfile $(location report_v2.txt) && " +
"sed -i'.original' 's/_TEST_VERSION = 1/_TEST_VERSION = 2/g' $(location test_file_v2_0.py)"),
tools = [":tf_upgrade_v2"],
)
py_test(
name = "test_file_v1_12",
size = "medium",
srcs = ["testdata/test_file_v1_12.py"],
strict_deps = True,
tags = [
"no_windows",
"v1only",
],
deps = [
"//tensorflow:tensorflow_py_no_contrib",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
],
)
py_test(
name = "test_file_v2_0",
size = "medium",
srcs = ["test_file_v2_0.py"],
strict_deps = True,
deps = [
"//tensorflow:tensorflow_py",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
],
)
exports_files(
[
"all_renames_v2.py",
"ast_edits.py",
"tf_upgrade.py",
"renames_v2.py",
"testdata/test_file_v0_11.py",
"testdata/test_file_v1_12.py",
],
)
tensorflow/tools/compatibility/README.md
+77
View File
@@ -0,0 +1,77 @@
# TensorFlow Python API Upgrade Utility
This tool allows you to upgrade your existing TensorFlow Python scripts,
specifically:
* `tf_upgrade_v2.py`: Upgrade code from TensorFlow 1.x to TensorFlow 2.0 preview.
* `tf_upgrade.py`: Upgrade code to TensorFlow 1.0 from TensorFlow 0.11.
## Running the script from pip package
First, install TensorFlow pip package*. See
https://www.tensorflow.org/install/pip.
Upgrade script can be run on a single Python file:
```
tf_upgrade_v2 --infile foo.py --outfile foo-upgraded.py
```
It will print a list of errors it finds that it can't fix. You can also run
it on a directory tree:
```
# upgrade the .py files and copy all the other files to the outtree
tf_upgrade_v2 --intree coolcode --outtree coolcode-upgraded
# just upgrade the .py files
tf_upgrade_v2 --intree coolcode --outtree coolcode-upgraded --copyotherfiles False
```
*Note: `tf_upgrade_v2` is installed automatically as a script by the pip install
after TensorFlow 1.12.
## Report
The script will also dump out a report e.g. which will detail changes
e.g.:
```
'tensorflow/tools/compatibility/testdata/test_file_v1_12.py' Line 65
--------------------------------------------------------------------------------
Added keyword 'input' to reordered function 'tf.argmax'
Renamed keyword argument from 'dimension' to 'axis'
Old: tf.argmax([[1, 3, 2]], dimension=0)
~~~~~~~~~~
New: tf.argmax(input=[[1, 3, 2]], axis=0)
```
## Caveats
- Don't update parts of your code manually before running this script. In
particular, functions that have had reordered arguments like `tf.argmax`
or `tf.batch_to_space` will cause the script to incorrectly add keyword
arguments that mismap arguments.
- This script wouldn't actually reorder arguments. Instead, the script will add
keyword arguments to functions that had their arguments reordered.
- The script assumes that `tensorflow` is imported using `import tensorflow as tf`.
- Note for upgrading to 2.0: Check out [tf2up.ml](http://tf2up.ml) for a
convenient tool to upgrade Jupyter notebooks and Python files in a GitHub
repository.
- Note for upgrading to 1.0: There are some syntaxes that are not handleable with this script as this
script was designed to use only standard python packages.
If the script fails with "A necessary keyword argument failed to be inserted." or
"Failed to find keyword lexicographically. Fix manually.", you can try
[@machrisaa's fork of this script](https://github.com/machrisaa/tf0to1).
[@machrisaa](https://github.com/machrisaa) has used the
[RedBaron Python refactoring engine](https://redbaron.readthedocs.io/en/latest/)
which is able to localize syntactic elements more reliably than the built-in
`ast` module this script is based upon. Note that the alternative script is not
available for TensorFlow 2.0 upgrade.
tensorflow/tools/compatibility/all_renames_v2.py
+517
View File
@@ -0,0 +1,517 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Provides a list of renames between TensorFlow 1.* and 2.0."""
from tensorflow.tools.compatibility import renames_v2
# pylint: disable=line-too-long
# Add additional renames not in renames_v2.py here.
# IMPORTANT: For the renames in here, if you also need to add to
# function_reorders or function_keyword_renames in tf_upgrade_v2.py,
# use the OLD function name.
# These renames happen after the arguments have been processed.
# After modifying this dict, run the following to update reorders_v2.py:
# bazel run tensorflow/tools/compatibility/update:generate_v2_reorders_map
manual_symbol_renames = {
"tf.batch_to_space_nd": "tf.batch_to_space",
"tf.batch_gather": "tf.compat.v1.batch_gather",
"tf.space_to_batch_nd": "tf.space_to_batch",
"tf.nn.space_to_batch": "tf.space_to_batch",
"tf.extract_image_patches": "tf.image.extract_patches",
"tf.image.extract_image_patches": "tf.image.extract_patches",
"tf.gfile.Copy": "tf.io.gfile.copy",
"tf.gfile.DeleteRecursively": "tf.io.gfile.rmtree",
"tf.gfile.Exists": "tf.io.gfile.exists",
"tf.gfile.Glob": "tf.io.gfile.glob",
"tf.gfile.GFile": "tf.io.gfile.GFile",
"tf.gfile.IsDirectory": "tf.io.gfile.isdir",
"tf.gfile.ListDirectory": "tf.io.gfile.listdir",
"tf.gfile.MakeDirs": "tf.io.gfile.makedirs",
"tf.gfile.MkDir": "tf.io.gfile.mkdir",
"tf.gfile.Open": "tf.io.gfile.GFile",
"tf.gfile.Remove": "tf.io.gfile.remove",
"tf.gfile.Rename": "tf.io.gfile.rename",
"tf.gfile.Stat": "tf.io.gfile.stat",
"tf.gfile.Walk": "tf.io.gfile.walk",
"tf.contrib.cluster_resolver.ClusterResolver": (
"tf.distribute.cluster_resolver.ClusterResolver"
),
"tf.contrib.cluster_resolver.GceClusterResolver": (
"tf.distribute.cluster_resolver.GCEClusterResolver"
),
"tf.contrib.cluster_resolver.KubernetesClusterResolver": (
"tf.distribute.cluster_resolver.KubernetesClusterResolver"
),
"tf.contrib.cluster_resolver.SimpleClusterResolver": (
"tf.distribute.cluster_resolver.SimpleClusterResolver"
),
"tf.contrib.cluster_resolver.SlurmClusterResolver": (
"tf.distribute.cluster_resolver.SlurmClusterResolver"
),
"tf.contrib.cluster_resolver.TFConfigClusterResolver": (
"tf.distribute.cluster_resolver.TFConfigClusterResolver"
),
"tf.contrib.cluster_resolver.TPUClusterResolver": (
"tf.distribute.cluster_resolver.TPUClusterResolver"
),
"tf.contrib.cluster_resolver.UnionClusterResolver": (
"tf.distribute.cluster_resolver.UnionClusterResolver"
),
"tf.contrib.data.AUTOTUNE": "tf.data.experimental.AUTOTUNE",
"tf.contrib.data.Counter": "tf.data.experimental.Counter",
"tf.contrib.data.CheckpointInputPipelineHook": (
"tf.data.experimental.CheckpointInputPipelineHook"
),
"tf.contrib.data.CsvDataset": "tf.data.experimental.CsvDataset",
"tf.contrib.data.Optional": "tf.data.experimental.Optional",
"tf.contrib.data.RandomDataset": "tf.data.experimental.RandomDataset",
"tf.contrib.data.Reducer": "tf.data.experimental.Reducer",
"tf.contrib.data.SqlDataset": "tf.data.experimental.SqlDataset",
"tf.contrib.data.StatsAggregator": "tf.data.experimental.StatsAggregator",
"tf.contrib.data.TFRecordWriter": "tf.data.experimental.TFRecordWriter",
"tf.contrib.data.assert_element_shape": (
"tf.data.experimental.assert_element_shape"
),
"tf.contrib.data.bucket_by_sequence_length": (
"tf.data.experimental.bucket_by_sequence_length"
),
"tf.contrib.data.choose_from_datasets": (
"tf.data.experimental.choose_from_datasets"
),
"tf.contrib.data.copy_to_device": "tf.data.experimental.copy_to_device",
"tf.contrib.data.dense_to_sparse_batch": (
"tf.data.experimental.dense_to_sparse_batch"
),
"tf.contrib.data.enumerate_dataset": (
"tf.data.experimental.enumerate_dataset"
),
"tf.contrib.data.get_next_as_optional": (
"tf.data.experimental.get_next_as_optional"
),
"tf.contrib.data.get_single_element": (
"tf.data.experimental.get_single_element"
),
"tf.contrib.data.group_by_reducer": "tf.data.experimental.group_by_reducer",
"tf.contrib.data.group_by_window": "tf.data.experimental.group_by_window",
"tf.contrib.data.ignore_errors": "tf.data.experimental.ignore_errors",
"tf.contrib.data.latency_stats": "tf.data.experimental.latency_stats",
"tf.contrib.data.make_batched_features_dataset": (
"tf.data.experimental.make_batched_features_dataset"
),
"tf.contrib.data.make_csv_dataset": "tf.data.experimental.make_csv_dataset",
"tf.contrib.data.make_saveable_from_iterator": (
"tf.data.experimental.make_saveable_from_iterator"
),
"tf.contrib.data.map_and_batch": "tf.data.experimental.map_and_batch",
"tf.contrib.data.parallel_interleave": (
"tf.data.experimental.parallel_interleave"
),
"tf.contrib.data.parse_example_dataset": (
"tf.data.experimental.parse_example_dataset"
),
"tf.contrib.data.prefetch_to_device": (
"tf.data.experimental.prefetch_to_device"
),
"tf.contrib.data.rejection_resample": (
"tf.data.experimental.rejection_resample"
),
"tf.contrib.data.sample_from_datasets": (
"tf.data.experimental.sample_from_datasets"
),
"tf.contrib.data.scan": "tf.data.experimental.scan",
"tf.contrib.data.set_stats_aggregator": (
"tf.data.experimental.set_stats_aggregator"
),
"tf.contrib.data.shuffle_and_repeat": (
"tf.data.experimental.shuffle_and_repeat"
),
"tf.contrib.data.unbatch": "tf.data.experimental.unbatch",
"tf.contrib.data.unique": "tf.data.experimental.unique",
"tf.contrib.distribute.CrossDeviceOps": "tf.distribute.CrossDeviceOps",
"tf.contrib.distribute.ReductionToOneDeviceCrossDeviceOps": (
"tf.distribute.ReductionToOneDevice"
),
"tf.contrib.framework.CriticalSection": "tf.CriticalSection",
"tf.contrib.framework.is_tensor": "tf.is_tensor",
"tf.contrib.framework.load_variable": "tf.train.load_variable",
"tf.contrib.framework.nest.assert_same_structure": (
"tf.nest.assert_same_structure"
),
"tf.contrib.framework.nest.flatten": "tf.nest.flatten",
"tf.contrib.framework.nest.is_nested": "tf.nest.is_nested",
"tf.contrib.framework.nest.map_structure": "tf.nest.map_structure",
"tf.contrib.framework.nest.pack_sequence_as": "tf.nest.pack_sequence_as",
"tf.contrib.batching.batch_function": "tf.nondifferentiable_batch_function",
"tf.contrib.util.constant_value": "tf.get_static_value",
"tf.contrib.saved_model.load_keras_model": (
"tf.compat.v1.keras.experimental.load_from_saved_model"
),
"tf.contrib.saved_model.save_keras_model": (
"tf.compat.v1.keras.experimental.export_saved_model"
),
"tf.contrib.rnn.RNNCell": "tf.compat.v1.nn.rnn_cell.RNNCell",
"tf.contrib.rnn.LSTMStateTuple": "tf.nn.rnn_cell.LSTMStateTuple",
"tf.contrib.rnn.BasicLSTMCell": "tf.compat.v1.nn.rnn_cell.BasicLSTMCell",
"tf.contrib.rnn.BasicRNNCell": "tf.compat.v1.nn.rnn_cell.BasicRNNCell",
"tf.contrib.rnn.GRUCell": "tf.compat.v1.nn.rnn_cell.GRUCell",
"tf.contrib.rnn.LSTMCell": "tf.compat.v1.nn.rnn_cell.LSTMCell",
"tf.contrib.rnn.MultiRNNCell": "tf.compat.v1.nn.rnn_cell.MultiRNNCell",
"tf.contrib.rnn.static_rnn": "tf.compat.v1.nn.static_rnn",
"tf.contrib.rnn.static_state_saving_rnn": (
"tf.compat.v1.nn.static_state_saving_rnn"
),
"tf.contrib.rnn.static_bidirectional_rnn": (
"tf.compat.v1.nn.static_bidirectional_rnn"
),
"tf.contrib.framework.sort": "tf.sort",
"tf.contrib.framework.argsort": "tf.argsort",
"tf.contrib.summary.all_summary_ops": (
"tf.compat.v1.summary.all_v2_summary_ops"
),
"tf.contrib.summary.always_record_summaries": (
"tf.compat.v2.summary.record_if"
),
"tf.contrib.summary.audio": "tf.compat.v2.summary.audio",
"tf.contrib.summary.create_file_writer": (
"tf.compat.v2.summary.create_file_writer"
),
"tf.contrib.summary.flush": "tf.compat.v2.summary.flush",
"tf.contrib.summary.generic": "tf.compat.v2.summary.write",
"tf.contrib.summary.histogram": "tf.compat.v2.summary.histogram",
"tf.contrib.summary.image": "tf.compat.v2.summary.image",
"tf.contrib.summary.initialize": "tf.compat.v1.summary.initialize",
"tf.contrib.summary.never_record_summaries": (
"tf.compat.v2.summary.record_if"
),
"tf.contrib.summary.scalar": "tf.compat.v2.summary.scalar",
"tf.contrib.tpu.CrossShardOptimizer": (
"tf.compat.v1.tpu.CrossShardOptimizer"
),
"tf.contrib.tpu.batch_parallel": "tf.compat.v1.tpu.batch_parallel",
"tf.contrib.tpu.bfloat16_scope": "tf.compat.v1.tpu.bfloat16_scope",
"tf.contrib.tpu.core": "tf.compat.v1.tpu.core",
"tf.contrib.tpu.cross_replica_sum": "tf.compat.v1.tpu.cross_replica_sum",
"tf.contrib.tpu.initialize_system": "tf.compat.v1.tpu.initialize_system",
"tf.contrib.tpu.outside_compilation": (
"tf.compat.v1.tpu.outside_compilation"
),
"tf.contrib.tpu.replicate": "tf.compat.v1.tpu.replicate",
"tf.contrib.tpu.rewrite": "tf.compat.v1.tpu.rewrite",
"tf.contrib.tpu.shard": "tf.compat.v1.tpu.shard",
"tf.contrib.tpu.shutdown_system": "tf.compat.v1.tpu.shutdown_system",
"tf.contrib.training.checkpoints_iterator": "tf.train.checkpoints_iterator",
"tf.contrib.layers.recompute_grad": "tf.recompute_grad",
"tf.count_nonzero": "tf.math.count_nonzero",
"tf.decode_raw": "tf.io.decode_raw",
"tf.manip.batch_to_space_nd": "tf.batch_to_space",
"tf.quantize_v2": "tf.quantization.quantize",
"tf.sparse_matmul": "tf.linalg.matmul",
"tf.random.stateless_multinomial": "tf.random.stateless_categorical",
"tf.substr": "tf.strings.substr",
# TODO(b/129398290)
"tf.string_split": "tf.compat.v1.string_split",
"tf.string_to_hash_bucket": "tf.strings.to_hash_bucket",
"tf.string_to_number": "tf.strings.to_number",
"tf.multinomial": "tf.random.categorical",
"tf.random.multinomial": "tf.random.categorical",
"tf.reduce_join": "tf.strings.reduce_join",
"tf.load_file_system_library": "tf.load_library",
"tf.bincount": "tf.math.bincount",
"tf.confusion_matrix": "tf.math.confusion_matrix",
"tf.train.confusion_matrix": "tf.math.confusion_matrix",
"tf.train.sdca_fprint": "tf.raw_ops.SdcaFprint",
"tf.train.sdca_optimizer": "tf.raw_ops.SdcaOptimizer",
"tf.train.sdca_shrink_l1": "tf.raw_ops.SdcaShrinkL1",
"tf.decode_csv": "tf.io.decode_csv",
"tf.data.Iterator": "tf.compat.v1.data.Iterator",
"tf.data.experimental.DatasetStructure": "tf.data.DatasetSpec",
"tf.data.experimental.OptionalStructure": "tf.OptionalSpec",
"tf.data.experimental.RaggedTensorStructure": "tf.RaggedTensorSpec",
"tf.data.experimental.SparseTensorStructure": "tf.SparseTensorSpec",
"tf.data.experimental.Structure": "tf.TypeSpec",
"tf.data.experimental.TensorArrayStructure": "tf.TensorArraySpec",
"tf.data.experimental.TensorStructure": "tf.TensorSpec",
"tf.parse_example": "tf.io.parse_example",
"tf.parse_single_example": "tf.io.parse_single_example",
"tf.nn.fused_batch_norm": "tf.compat.v1.nn.fused_batch_norm",
"tf.nn.softmax_cross_entropy_with_logits_v2": (
"tf.nn.softmax_cross_entropy_with_logits"
),
"tf.losses.Reduction.MEAN": "tf.compat.v1.losses.Reduction.MEAN",
"tf.losses.Reduction.SUM_BY_NONZERO_WEIGHTS": (
"tf.compat.v1.losses.Reduction.SUM_BY_NONZERO_WEIGHTS"
),
"tf.losses.Reduction.SUM_OVER_NONZERO_WEIGHTS": (
"tf.compat.v1.losses.Reduction.SUM_OVER_NONZERO_WEIGHTS"
),
"tf.lite.constants.FLOAT": "tf.float32",
"tf.lite.constants.FLOAT16": "tf.float16",
"tf.lite.constants.INT16": "tf.int16",
"tf.lite.constants.INT32": "tf.int32",
"tf.lite.constants.INT64": "tf.int64",
"tf.lite.constants.INT8": "tf.int8",
"tf.lite.constants.STRING": "tf.string",
"tf.lite.constants.QUANTIZED_UINT8": "tf.uint8",
"tf.arg_max": "tf.argmax",
"tf.arg_min": "tf.argmin",
# tf.nn.ctc_loss is still available in 2.0 but behavior
# changed significantly.
"tf.nn.ctc_loss": "tf.compat.v1.nn.ctc_loss",
# tf.saved_model.load in 1.x has no equivalent in 2.x, but there is a
# symbol with the same name.
"tf.saved_model.load": "tf.compat.v1.saved_model.load",
"tf.saved_model.loader.load": "tf.compat.v1.saved_model.load",
"tf.saved_model.load_v2": "tf.compat.v2.saved_model.load",
"tf.image.resize_images": "tf.image.resize",
"tf.assert_equal": "tf.compat.v1.assert_equal",
"tf.assert_greater": "tf.compat.v1.assert_greater",
"tf.assert_greater_equal": "tf.compat.v1.assert_greater_equal",
"tf.assert_integer": "tf.compat.v1.assert_integer",
"tf.assert_less": "tf.compat.v1.assert_less",
"tf.assert_less_equal": "tf.compat.v1.assert_less_equal",
"tf.assert_near": "tf.compat.v1.assert_near",
"tf.assert_negative": "tf.compat.v1.assert_negative",
"tf.assert_non_negative": "tf.compat.v1.assert_non_negative",
"tf.assert_non_positive": "tf.compat.v1.assert_non_positive",
"tf.assert_none_equal": "tf.compat.v1.assert_none_equal",
"tf.assert_positive": "tf.compat.v1.assert_positive",
"tf.assert_rank": "tf.compat.v1.assert_rank",
"tf.assert_rank_at_least": "tf.compat.v1.assert_rank_at_least",
"tf.assert_rank_in": "tf.compat.v1.assert_rank_in",
"tf.assert_scalar": "tf.compat.v1.assert_scalar",
"tf.assert_type": "tf.compat.v1.assert_type",
"tf.assert_variables_initialized": (
"tf.compat.v1.assert_variables_initialized"
),
"tf.debugging.assert_equal": "tf.compat.v1.debugging.assert_equal",
"tf.debugging.assert_greater": "tf.compat.v1.debugging.assert_greater",
"tf.debugging.assert_greater_equal": (
"tf.compat.v1.debugging.assert_greater_equal"
),
"tf.debugging.assert_integer": "tf.compat.v1.debugging.assert_integer",
"tf.debugging.assert_less": "tf.compat.v1.debugging.assert_less",
"tf.debugging.assert_less_equal": (
"tf.compat.v1.debugging.assert_less_equal"
),
"tf.debugging.assert_near": "tf.compat.v1.debugging.assert_near",
"tf.debugging.assert_negative": "tf.compat.v1.debugging.assert_negative",
"tf.debugging.assert_non_negative": (
"tf.compat.v1.debugging.assert_non_negative"
),
"tf.debugging.assert_non_positive": (
"tf.compat.v1.debugging.assert_non_positive"
),
"tf.debugging.assert_none_equal": (
"tf.compat.v1.debugging.assert_none_equal"
),
"tf.debugging.assert_positive": "tf.compat.v1.debugging.assert_positive",
"tf.debugging.assert_rank": "tf.compat.v1.debugging.assert_rank",
"tf.debugging.assert_rank_at_least": (
"tf.compat.v1.debugging.assert_rank_at_least"
),
"tf.debugging.assert_rank_in": "tf.compat.v1.debugging.assert_rank_in",
"tf.debugging.assert_scalar": "tf.compat.v1.debugging.assert_scalar",
"tf.debugging.assert_type": "tf.compat.v1.debugging.assert_type",
"tf.errors.exception_type_from_error_code": (
"tf.compat.v1.errors.exception_type_from_error_code"
),
"tf.errors.error_code_from_exception_type": (
"tf.compat.v1.errors.error_code_from_exception_type"
),
"tf.errors.raise_exception_on_not_ok_status": (
"tf.compat.v1.errors.raise_exception_on_not_ok_status"
),
"tf.nn.max_pool": "tf.nn.max_pool2d",
"tf.nn.avg_pool": "tf.nn.avg_pool2d",
"tf.keras.initializers.zeros": "tf.compat.v1.keras.initializers.zeros",
"tf.keras.initializers.Zeros": "tf.compat.v1.keras.initializers.Zeros",
"tf.keras.initializers.ones": "tf.compat.v1.keras.initializers.ones",
"tf.keras.initializers.Ones": "tf.compat.v1.keras.initializers.Ones",
"tf.keras.initializers.constant": (
"tf.compat.v1.keras.initializers.constant"
),
"tf.keras.initializers.Constant": (
"tf.compat.v1.keras.initializers.Constant"
),
"tf.keras.initializers.VarianceScaling": (
"tf.compat.v1.keras.initializers.VarianceScaling"
),
"tf.keras.initializers.Orthogonal": (
"tf.compat.v1.keras.initializers.Orthogonal"
),
"tf.keras.initializers.orthogonal": (
"tf.compat.v1.keras.initializers.orthogonal"
),
"tf.keras.initializers.Identity": (
"tf.compat.v1.keras.initializers.Identity"
),
"tf.keras.initializers.identity": (
"tf.compat.v1.keras.initializers.identity"
),
"tf.keras.initializers.glorot_uniform": (
"tf.compat.v1.keras.initializers.glorot_uniform"
),
"tf.keras.initializers.glorot_normal": (
"tf.compat.v1.keras.initializers.glorot_normal"
),
"tf.keras.initializers.lecun_normal": (
"tf.compat.v1.keras.initializers.lecun_normal"
),
"tf.keras.initializers.lecun_uniform": (
"tf.compat.v1.keras.initializers.lecun_uniform"
),
"tf.keras.initializers.he_normal": (
"tf.compat.v1.keras.initializers.he_normal"
),
"tf.keras.initializers.he_uniform": (
"tf.compat.v1.keras.initializers.he_uniform"
),
"tf.keras.initializers.TruncatedNormal": (
"tf.compat.v1.keras.initializers.TruncatedNormal"
),
"tf.keras.initializers.truncated_normal": (
"tf.compat.v1.keras.initializers.truncated_normal"
),
"tf.keras.initializers.RandomUniform": (
"tf.compat.v1.keras.initializers.RandomUniform"
),
"tf.keras.initializers.uniform": "tf.compat.v1.keras.initializers.uniform",
"tf.keras.initializers.random_uniform": (
"tf.compat.v1.keras.initializers.random_uniform"
),
"tf.keras.initializers.RandomNormal": (
"tf.compat.v1.keras.initializers.RandomNormal"
),
"tf.keras.initializers.normal": "tf.compat.v1.keras.initializers.normal",
"tf.keras.initializers.random_normal": (
"tf.compat.v1.keras.initializers.random_normal"
),
"tf.zeros_initializer": "tf.compat.v1.zeros_initializer",
"tf.initializers.zeros": "tf.compat.v1.initializers.zeros",
"tf.ones_initializer": "tf.compat.v1.ones_initializer",
"tf.initializers.ones": "tf.compat.v1.initializers.ones",
"tf.constant_initializer": "tf.compat.v1.constant_initializer",
"tf.initializers.constant": "tf.compat.v1.initializers.constant",
"tf.random_uniform_initializer": "tf.compat.v1.random_uniform_initializer",
"tf.initializers.random_uniform": (
"tf.compat.v1.initializers.random_uniform"
),
"tf.random_normal_initializer": "tf.compat.v1.random_normal_initializer",
"tf.initializers.random_normal": "tf.compat.v1.initializers.random_normal",
"tf.truncated_normal_initializer": (
"tf.compat.v1.truncated_normal_initializer"
),
"tf.initializers.truncated_normal": (
"tf.compat.v1.initializers.truncated_normal"
),
"tf.variance_scaling_initializer": (
"tf.compat.v1.variance_scaling_initializer"
),
"tf.initializers.variance_scaling": (
"tf.compat.v1.initializers.variance_scaling"
),
"tf.orthogonal_initializer": "tf.compat.v1.orthogonal_initializer",
"tf.initializers.orthogonal": "tf.compat.v1.initializers.orthogonal",
"tf.glorot_uniform_initializer": "tf.compat.v1.glorot_uniform_initializer",
"tf.initializers.glorot_uniform": (
"tf.compat.v1.initializers.glorot_uniform"
),
"tf.glorot_normal_initializer": "tf.compat.v1.glorot_normal_initializer",
"tf.initializers.glorot_normal": "tf.compat.v1.initializers.glorot_normal",
"tf.initializers.identity": "tf.compat.v1.initializers.identity",
"tf.initializers.lecun_normal": "tf.compat.v1.initializers.lecun_normal",
"tf.initializers.lecun_uniform": "tf.compat.v1.initializers.lecun_uniform",
"tf.initializers.he_normal": "tf.compat.v1.initializers.he_normal",
"tf.initializers.he_uniform": "tf.compat.v1.initializers.he_uniform",
"tf.data.experimental.map_and_batch_with_legacy_function": (
"tf.compat.v1.data.experimental.map_and_batch_with_legacy_function"
),
"tf.nn.conv2d_backprop_input": "tf.nn.conv2d_transpose",
"tf.test.compute_gradient": "tf.compat.v1.test.compute_gradient",
"tf.floor_div": "tf.math.floordiv",
"tf.where": "tf.compat.v1.where",
"tf.where_v2": "tf.compat.v2.where",
"tf.app.flags": "tf.compat.v1.app.flags",
}
# pylint: enable=line-too-long
def add_contrib_direct_import_support(symbol_dict):
"""Add support for `tf.contrib.*` alias `contrib_*.` Updates dict in place."""
for symbol_name in list(symbol_dict.keys()):
symbol_alias = symbol_name.replace("tf.contrib.", "contrib_")
symbol_dict[symbol_alias] = symbol_dict[symbol_name]
add_contrib_direct_import_support(manual_symbol_renames)
symbol_renames = renames_v2.renames
symbol_renames.update(manual_symbol_renames)
addons_symbol_mappings = {
"tf.contrib.layers.poincare_normalize":
"tfa.layers.PoincareNormalize",
"tf.contrib.layers.maxout":
"tfa.layers.Maxout",
"tf.contrib.layers.group_norm":
"tfa.layers.GroupNormalization",
"tf.contrib.layers.instance_norm":
"tfa.layers.InstanceNormalization",
"tf.contrib.sparsemax.sparsemax":
"tfa.activations.sparsemax",
"tf.contrib.losses.metric_learning.contrastive_loss":
"tfa.losses.ContrastiveLoss",
"tf.contrib.losses.metric_learning.lifted_struct_loss":
"tfa.losses.LiftedStructLoss",
"tf.contrib.sparsemax.sparsemax_loss":
"tfa.losses.SparsemaxLoss",
"tf.contrib.losses.metric_learning.triplet_semihard_loss":
"tfa.losses.TripletSemiHardLoss",
"tf.contrib.opt.LazyAdamOptimizer":
"tfa.optimizers.LazyAdam",
"tf.contrib.opt.MovingAverageOptimizer":
"tfa.optimizers.MovingAverage",
"tf.contrib.opt.MomentumWOptimizer":
"tfa.optimizers.SGDW",
"tf.contrib.opt.AdamWOptimizer":
"tfa.optimizers.AdamW",
"tf.contrib.opt.extend_with_decoupled_weight_decay":
"tfa.optimizers.extend_with_decoupled_weight_decay",
"tf.contrib.text.skip_gram_sample":
"tfa.text.skip_gram_sample",
"tf.contrib.text.skip_gram_sample_with_text_vocab":
"tfa.text.skip_gram_sample_with_text_vocab",
"tf.contrib.image.dense_image_warp":
"tfa.image.dense_image_warp",
"tf.contrib.image.adjust_hsv_in_yiq":
"tfa.image.adjust_hsv_in_yiq",
"tf.contrib.image.compose_transforms":
"tfa.image.compose_transforms",
"tf.contrib.image.random_hsv_in_yiq":
"tfa.image.random_hsv_in_yiq",
"tf.contrib.image.angles_to_projective_transforms":
"tfa.image.angles_to_projective_transforms",
"tf.contrib.image.matrices_to_flat_transforms":
"tfa.image.matrices_to_flat_transforms",
"tf.contrib.image.rotate":
"tfa.image.rotate",
"tf.contrib.image.transform":
"tfa.image.transform",
"tf.contrib.rnn.NASCell":
"tfa.rnn.NASCell",
"tf.contrib.rnn.LayerNormBasicLSTMCell":
"tfa.rnn.LayerNormLSTMCell"
}
add_contrib_direct_import_support(addons_symbol_mappings)
tensorflow/tools/compatibility/all_renames_v2_test.py
+34
View File
@@ -0,0 +1,34 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for all_renames_v2."""
from tensorflow.python.framework import test_util
from tensorflow.python.platform import test as test_lib
from tensorflow.tools.compatibility import all_renames_v2
class AllRenamesV2Test(test_util.TensorFlowTestCase):
def test_no_identity_renames(self):
identity_renames = [
old_name
for old_name, new_name in all_renames_v2.symbol_renames.items()
if old_name == new_name
]
self.assertEmpty(identity_renames)
if __name__ == "__main__":
test_lib.main()
tensorflow/tools/compatibility/ast_edits.py
+1188
View File
File diff suppressed because it is too large Load Diff
tensorflow/tools/compatibility/ast_edits_test.py
+730
View File
@@ -0,0 +1,730 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for ast_edits which is used in tf upgraders.
All of the tests assume that we want to change from an API containing
import foo as f
def f(a, b, kw1, kw2): ...
def g(a, b, kw1, c, kw1_alias): ...
def g2(a, b, kw1, c, d, kw1_alias): ...
def h(a, kw1, kw2, kw1_alias, kw2_alias): ...
and the changes to the API consist of renaming, reordering, and/or removing
arguments. Thus, we want to be able to generate changes to produce each of the
following new APIs:
import bar as f
def f(a, b, kw1, kw3): ...
def f(a, b, kw2, kw1): ...
def f(a, b, kw3, kw1): ...
def g(a, b, kw1, c): ...
def g(a, b, c, kw1): ...
def g2(a, b, kw1, c, d): ...
def g2(a, b, c, d, kw1): ...
def h(a, kw1, kw2): ...
"""
import ast
import io
import os
import sys
import unittest
import pasta
from tensorflow.python.framework import test_util
from tensorflow.python.platform import test as test_lib
from tensorflow.tools.compatibility import ast_edits
class ModuleDeprecationSpec(ast_edits.NoUpdateSpec):
"""A specification which deprecates 'a.b'."""
def __init__(self):
ast_edits.NoUpdateSpec.__init__(self)
self.module_deprecations.update({"a.b": (ast_edits.ERROR, "a.b is evil.")})
class RenameKeywordSpec(ast_edits.NoUpdateSpec):
"""A specification where kw2 gets renamed to kw3.
The new API is
def f(a, b, kw1, kw3): ...
"""
def __init__(self):
ast_edits.NoUpdateSpec.__init__(self)
self.update_renames()
def update_renames(self):
self.function_keyword_renames["f"] = {"kw2": "kw3"}
class ReorderKeywordSpec(ast_edits.NoUpdateSpec):
"""A specification where kw2 gets moved in front of kw1.
The new API is
def f(a, b, kw2, kw1): ...
"""
def __init__(self):
ast_edits.NoUpdateSpec.__init__(self)
self.update_reorders()
def update_reorders(self):
# Note that these should be in the old order.
self.function_reorders["f"] = ["a", "b", "kw1", "kw2"]
class ReorderAndRenameKeywordSpec(ReorderKeywordSpec, RenameKeywordSpec):
"""A specification where kw2 gets moved in front of kw1 and is changed to kw3.
The new API is
def f(a, b, kw3, kw1): ...
"""
def __init__(self):
ReorderKeywordSpec.__init__(self)
RenameKeywordSpec.__init__(self)
self.update_renames()
self.update_reorders()
class RemoveDeprecatedAliasKeyword(ast_edits.NoUpdateSpec):
"""A specification where kw1_alias is removed in g.
The new API is
def g(a, b, kw1, c): ...
def g2(a, b, kw1, c, d): ...
"""
def __init__(self):
ast_edits.NoUpdateSpec.__init__(self)
self.function_keyword_renames["g"] = {"kw1_alias": "kw1"}
self.function_keyword_renames["g2"] = {"kw1_alias": "kw1"}
class RemoveDeprecatedAliasAndReorderRest(RemoveDeprecatedAliasKeyword):
"""A specification where kw1_alias is removed in g.
The new API is
def g(a, b, c, kw1): ...
def g2(a, b, c, d, kw1): ...
"""
def __init__(self):
RemoveDeprecatedAliasKeyword.__init__(self)
# Note that these should be in the old order.
self.function_reorders["g"] = ["a", "b", "kw1", "c"]
self.function_reorders["g2"] = ["a", "b", "kw1", "c", "d"]
class RemoveMultipleKeywordArguments(ast_edits.NoUpdateSpec):
"""A specification where both keyword aliases are removed from h.
The new API is
def h(a, kw1, kw2): ...
"""
def __init__(self):
ast_edits.NoUpdateSpec.__init__(self)
self.function_keyword_renames["h"] = {
"kw1_alias": "kw1",
"kw2_alias": "kw2",
}
class RenameImports(ast_edits.NoUpdateSpec):
"""Specification for renaming imports."""
def __init__(self):
ast_edits.NoUpdateSpec.__init__(self)
self.import_renames = {
"foo": ast_edits.ImportRename(
"bar",
excluded_prefixes=["foo.baz"])
}
class TestAstEdits(test_util.TensorFlowTestCase):
def _upgrade(self, spec, old_file_text):
in_file = io.StringIO(old_file_text)
out_file = io.StringIO()
upgrader = ast_edits.ASTCodeUpgrader(spec)
count, report, errors = (
upgrader.process_opened_file("test.py", in_file,
"test_out.py", out_file))
return (count, report, errors), out_file.getvalue()
def testModuleDeprecation(self):
text = "a.b.c(a.b.x)"
(_, _, errors), new_text = self._upgrade(ModuleDeprecationSpec(), text)
self.assertEqual(text, new_text)
self.assertIn("Using member a.b.c", errors[0])
self.assertIn("1:0", errors[0])
self.assertIn("Using member a.b.c", errors[0])
self.assertIn("1:6", errors[1])
def testNoTransformIfNothingIsSupplied(self):
text = "f(a, b, kw1=c, kw2=d)\n"
_, new_text = self._upgrade(ast_edits.NoUpdateSpec(), text)
self.assertEqual(new_text, text)
text = "f(a, b, c, d)\n"
_, new_text = self._upgrade(ast_edits.NoUpdateSpec(), text)
self.assertEqual(new_text, text)
@unittest.skipIf(sys.version_info >= (3, 14), "Broken in 3.14")
def testGooglePastaRoundTripModuleDocstringAndImport(self):
text = '"""Tests for tf upgrader."""\n\nimport io\n'
self.assertEqual(pasta.dump(pasta.parse(text)), text)
def testGooglePastaRoundTripConstants(self):
text = (
"x = tf.concat(0, [x for x in y])\n"
"y = b'bytes'\n"
"z = (True, False, None, ...)\n"
)
self.assertEqual(pasta.dump(pasta.parse(text)), text)
def testGooglePastaRoundTripIndentedImport(self):
text = (
"\n"
"try:\n"
" import tensorflow as tf # import line\n"
"\n"
" tf.ones([4, 5])\n"
"except AttributeError:\n"
" pass\n"
)
self.assertEqual(pasta.dump(pasta.parse(text)), text)
def testKeywordRename(self):
"""Test that we get the expected result if renaming kw2 to kw3."""
text = "f(a, b, kw1=c, kw2=d)\n"
expected = "f(a, b, kw1=c, kw3=d)\n"
(_, report, _), new_text = self._upgrade(RenameKeywordSpec(), text)
self.assertEqual(new_text, expected)
self.assertNotIn("Manual check required", report)
# No keywords specified, no reordering, so we should get input as output
text = "f(a, b, c, d)\n"
(_, report, _), new_text = self._upgrade(RenameKeywordSpec(), text)
self.assertEqual(new_text, text)
self.assertNotIn("Manual check required", report)
# Positional *args passed in that we cannot inspect, should warn
text = "f(a, *args)\n"
(_, report, _), _ = self._upgrade(RenameKeywordSpec(), text)
self.assertNotIn("Manual check required", report)
# **kwargs passed in that we cannot inspect, should warn
text = "f(a, b, kw1=c, **kwargs)\n"
(_, report, _), _ = self._upgrade(RenameKeywordSpec(), text)
self.assertIn("Manual check required", report)
def testKeywordReorderWithParens(self):
"""Test that we get the expected result if there are parens around args."""
text = "f((a), ( ( b ) ))\n"
acceptable_outputs = [
# No change is a valid output
text,
# Also cases where all arguments are fully specified are allowed
"f(a=(a), b=( ( b ) ))\n",
# Making the parens canonical is ok
"f(a=(a), b=((b)))\n",
]
_, new_text = self._upgrade(ReorderKeywordSpec(), text)
self.assertIn(new_text, acceptable_outputs)
def testKeywordReorder(self):
"""Test that we get the expected result if kw2 is now before kw1."""
text = "f(a, b, kw1=c, kw2=d)\n"
acceptable_outputs = [
# No change is a valid output
text,
# Just reordering the kw.. args is also ok
"f(a, b, kw2=d, kw1=c)\n",
# Also cases where all arguments are fully specified are allowed
"f(a=a, b=b, kw1=c, kw2=d)\n",
"f(a=a, b=b, kw2=d, kw1=c)\n",
]
(_, report, _), new_text = self._upgrade(ReorderKeywordSpec(), text)
self.assertIn(new_text, acceptable_outputs)
self.assertNotIn("Manual check required", report)
# Keywords are reordered, so we should reorder arguments too
text = "f(a, b, c, d)\n"
acceptable_outputs = [
"f(a, b, d, c)\n",
"f(a=a, b=b, kw1=c, kw2=d)\n",
"f(a=a, b=b, kw2=d, kw1=c)\n",
]
(_, report, _), new_text = self._upgrade(ReorderKeywordSpec(), text)
self.assertIn(new_text, acceptable_outputs)
self.assertNotIn("Manual check required", report)
# Positional *args passed in that we cannot inspect, should warn
text = "f(a, b, *args)\n"
(_, report, _), _ = self._upgrade(ReorderKeywordSpec(), text)
self.assertIn("Manual check required", report)
# **kwargs passed in that we cannot inspect, should warn
text = "f(a, b, kw1=c, **kwargs)\n"
(_, report, _), _ = self._upgrade(ReorderKeywordSpec(), text)
self.assertNotIn("Manual check required", report)
def testKeywordReorderAndRename(self):
"""Test that we get the expected result if kw2 is renamed and moved."""
text = "f(a, b, kw1=c, kw2=d)\n"
acceptable_outputs = [
"f(a, b, kw3=d, kw1=c)\n",
"f(a=a, b=b, kw1=c, kw3=d)\n",
"f(a=a, b=b, kw3=d, kw1=c)\n",
]
(_, report, _), new_text = self._upgrade(
ReorderAndRenameKeywordSpec(), text)
self.assertIn(new_text, acceptable_outputs)
self.assertNotIn("Manual check required", report)
# Keywords are reordered, so we should reorder arguments too
text = "f(a, b, c, d)\n"
acceptable_outputs = [
"f(a, b, d, c)\n",
"f(a=a, b=b, kw1=c, kw3=d)\n",
"f(a=a, b=b, kw3=d, kw1=c)\n",
]
(_, report, _), new_text = self._upgrade(
ReorderAndRenameKeywordSpec(), text)
self.assertIn(new_text, acceptable_outputs)
self.assertNotIn("Manual check required", report)
# Positional *args passed in that we cannot inspect, should warn
text = "f(a, *args, kw1=c)\n"
(_, report, _), _ = self._upgrade(ReorderAndRenameKeywordSpec(), text)
self.assertIn("Manual check required", report)
# **kwargs passed in that we cannot inspect, should warn
text = "f(a, b, kw1=c, **kwargs)\n"
(_, report, _), _ = self._upgrade(ReorderAndRenameKeywordSpec(), text)
self.assertIn("Manual check required", report)
def testRemoveDeprecatedKeywordAlias(self):
"""Test that we get the expected result if a keyword alias is removed."""
text = "g(a, b, kw1=x, c=c)\n"
acceptable_outputs = [
# Not using deprecated alias, so original is ok
text,
"g(a=a, b=b, kw1=x, c=c)\n",
]
_, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
self.assertIn(new_text, acceptable_outputs)
# No keyword used, should be no change
text = "g(a, b, x, c)\n"
_, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
self.assertEqual(new_text, text)
# If we used the alias, it should get renamed
text = "g(a, b, kw1_alias=x, c=c)\n"
acceptable_outputs = [
"g(a, b, kw1=x, c=c)\n",
"g(a, b, c=c, kw1=x)\n",
"g(a=a, b=b, kw1=x, c=c)\n",
"g(a=a, b=b, c=c, kw1=x)\n",
]
_, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
self.assertIn(new_text, acceptable_outputs)
# It should get renamed even if it's last
text = "g(a, b, c=c, kw1_alias=x)\n"
acceptable_outputs = [
"g(a, b, kw1=x, c=c)\n",
"g(a, b, c=c, kw1=x)\n",
"g(a=a, b=b, kw1=x, c=c)\n",
"g(a=a, b=b, c=c, kw1=x)\n",
]
_, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
self.assertIn(new_text, acceptable_outputs)
def testRemoveDeprecatedKeywordAndReorder(self):
"""Test for when a keyword alias is removed and args are reordered."""
text = "g(a, b, kw1=x, c=c)\n"
acceptable_outputs = [
"g(a, b, c=c, kw1=x)\n",
"g(a=a, b=b, kw1=x, c=c)\n",
]
_, new_text = self._upgrade(RemoveDeprecatedAliasAndReorderRest(), text)
self.assertIn(new_text, acceptable_outputs)
# Keywords are reordered, so we should reorder arguments too
text = "g(a, b, x, c)\n"
# Don't accept an output which doesn't reorder c and d
acceptable_outputs = [
"g(a, b, c, x)\n",
"g(a=a, b=b, kw1=x, c=c)\n",
]
_, new_text = self._upgrade(RemoveDeprecatedAliasAndReorderRest(), text)
self.assertIn(new_text, acceptable_outputs)
# If we used the alias, it should get renamed
text = "g(a, b, kw1_alias=x, c=c)\n"
acceptable_outputs = [
"g(a, b, kw1=x, c=c)\n",
"g(a, b, c=c, kw1=x)\n",
"g(a=a, b=b, kw1=x, c=c)\n",
"g(a=a, b=b, c=c, kw1=x)\n",
]
_, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
self.assertIn(new_text, acceptable_outputs)
# It should get renamed and reordered even if it's last
text = "g(a, b, c=c, kw1_alias=x)\n"
acceptable_outputs = [
"g(a, b, kw1=x, c=c)\n",
"g(a, b, c=c, kw1=x)\n",
"g(a=a, b=b, kw1=x, c=c)\n",
"g(a=a, b=b, c=c, kw1=x)\n",
]
_, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
self.assertIn(new_text, acceptable_outputs)
def testRemoveDeprecatedKeywordAndReorder2(self):
"""Same as testRemoveDeprecatedKeywordAndReorder but on g2 (more args)."""
text = "g2(a, b, kw1=x, c=c, d=d)\n"
acceptable_outputs = [
"g2(a, b, c=c, d=d, kw1=x)\n",
"g2(a=a, b=b, kw1=x, c=c, d=d)\n",
]
_, new_text = self._upgrade(RemoveDeprecatedAliasAndReorderRest(), text)
self.assertIn(new_text, acceptable_outputs)
# Keywords are reordered, so we should reorder arguments too
text = "g2(a, b, x, c, d)\n"
# Don't accept an output which doesn't reorder c and d
acceptable_outputs = [
"g2(a, b, c, d, x)\n",
"g2(a=a, b=b, kw1=x, c=c, d=d)\n",
]
_, new_text = self._upgrade(RemoveDeprecatedAliasAndReorderRest(), text)
self.assertIn(new_text, acceptable_outputs)
# If we used the alias, it should get renamed
text = "g2(a, b, kw1_alias=x, c=c, d=d)\n"
acceptable_outputs = [
"g2(a, b, kw1=x, c=c, d=d)\n",
"g2(a, b, c=c, d=d, kw1=x)\n",
"g2(a=a, b=b, kw1=x, c=c, d=d)\n",
"g2(a=a, b=b, c=c, d=d, kw1=x)\n",
]
_, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
self.assertIn(new_text, acceptable_outputs)
# It should get renamed and reordered even if it's not in order
text = "g2(a, b, d=d, c=c, kw1_alias=x)\n"
acceptable_outputs = [
"g2(a, b, kw1=x, c=c, d=d)\n",
"g2(a, b, c=c, d=d, kw1=x)\n",
"g2(a, b, d=d, c=c, kw1=x)\n",
"g2(a=a, b=b, kw1=x, c=c, d=d)\n",
"g2(a=a, b=b, c=c, d=d, kw1=x)\n",
"g2(a=a, b=b, d=d, c=c, kw1=x)\n",
]
_, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
self.assertIn(new_text, acceptable_outputs)
def testRemoveMultipleKeywords(self):
"""Remove multiple keywords at once."""
# Not using deprecated keywords -> no rename
text = "h(a, kw1=x, kw2=y)\n"
_, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text)
self.assertEqual(new_text, text)
# Using positional arguments (in proper order) -> no change
text = "h(a, x, y)\n"
_, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text)
self.assertEqual(new_text, text)
# Use only the old names, in order
text = "h(a, kw1_alias=x, kw2_alias=y)\n"
acceptable_outputs = [
"h(a, x, y)\n",
"h(a, kw1=x, kw2=y)\n",
"h(a=a, kw1=x, kw2=y)\n",
"h(a, kw2=y, kw1=x)\n",
"h(a=a, kw2=y, kw1=x)\n",
]
_, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text)
self.assertIn(new_text, acceptable_outputs)
# Use only the old names, in reverse order, should give one of same outputs
text = "h(a, kw2_alias=y, kw1_alias=x)\n"
_, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text)
self.assertIn(new_text, acceptable_outputs)
# Mix old and new names
text = "h(a, kw1=x, kw2_alias=y)\n"
_, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text)
self.assertIn(new_text, acceptable_outputs)
def testUnrestrictedFunctionWarnings(self):
class FooWarningSpec(ast_edits.NoUpdateSpec):
"""Usages of function attribute foo() prints out a warning."""
def __init__(self):
ast_edits.NoUpdateSpec.__init__(self)
self.function_warnings = {"*.foo": (ast_edits.WARNING, "not good")}
texts = ["object.foo()", "get_object().foo()",
"get_object().foo()", "object.foo().bar()"]
for text in texts:
(_, report, _), _ = self._upgrade(FooWarningSpec(), text)
self.assertIn("not good", report)
# Note that foo() won't result in a warning, because in this case foo is
# not an attribute, but a name.
false_alarms = ["foo", "foo()", "foo.bar()", "obj.run_foo()", "obj.foo"]
for text in false_alarms:
(_, report, _), _ = self._upgrade(FooWarningSpec(), text)
self.assertNotIn("not good", report)
def testFullNameNode(self):
t = ast_edits.full_name_node("a.b.c")
self.assertEqual(
ast.dump(t),
"Attribute(value=Attribute(value=Name(id='a', ctx=Load()), attr='b', "
"ctx=Load()), attr='c', ctx=Load())")
def testImport(self):
# foo should be renamed to bar.
text = "import foo as f"
expected_text = "import bar as f"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(expected_text, new_text)
text = "import foo"
expected_text = "import bar as foo"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(expected_text, new_text)
text = "import foo.test"
expected_text = "import bar.test"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(expected_text, new_text)
text = "import foo.test as t"
expected_text = "import bar.test as t"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(expected_text, new_text)
text = "import foo as f, a as b"
expected_text = "import bar as f, a as b"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(expected_text, new_text)
def testFromImport(self):
# foo should be renamed to bar.
text = "from foo import a"
expected_text = "from bar import a"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(expected_text, new_text)
text = "from foo.a import b"
expected_text = "from bar.a import b"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(expected_text, new_text)
text = "from foo import *"
expected_text = "from bar import *"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(expected_text, new_text)
text = "from foo import a, b"
expected_text = "from bar import a, b"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(expected_text, new_text)
def testImport_NoChangeNeeded(self):
text = "import bar as b"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(text, new_text)
def testFromImport_NoChangeNeeded(self):
text = "from bar import a as b"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(text, new_text)
def testExcludedImport(self):
# foo.baz module is excluded from changes.
text = "import foo.baz"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(text, new_text)
text = "import foo.baz as a"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(text, new_text)
text = "from foo import baz as a"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(text, new_text)
text = "from foo.baz import a"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(text, new_text)
def testMultipleImports(self):
text = "import foo.bar as a, foo.baz as b, foo.baz.c, foo.d"
expected_text = "import bar.bar as a, foo.baz as b, foo.baz.c, bar.d"
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(expected_text, new_text)
text = "from foo import baz, a, c"
expected_text = """from foo import baz
from bar import a, c"""
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(expected_text, new_text)
def testImportInsideFunction(self):
text = """
def t():
from c import d
from foo import baz, a
from e import y
"""
expected_text = """
def t():
from c import d
from foo import baz
from bar import a
from e import y
"""
_, new_text = self._upgrade(RenameImports(), text)
self.assertEqual(expected_text, new_text)
def testUpgradeInplaceWithSymlink(self):
if os.name == "nt":
self.skipTest("os.symlink doesn't work uniformly on Windows.")
upgrade_dir = os.path.join(self.get_temp_dir(), "foo")
os.mkdir(upgrade_dir)
file_a = os.path.join(upgrade_dir, "a.py")
file_b = os.path.join(upgrade_dir, "b.py")
with open(file_a, "a") as f:
f.write("import foo as f")
os.symlink(file_a, file_b)
upgrader = ast_edits.ASTCodeUpgrader(RenameImports())
upgrader.process_tree_inplace(upgrade_dir)
self.assertTrue(os.path.islink(file_b))
self.assertEqual(file_a, os.readlink(file_b))
with open(file_a, "r") as f:
self.assertEqual("import bar as f", f.read())
def testUpgradeInPlaceWithSymlinkInDifferentDir(self):
if os.name == "nt":
self.skipTest("os.symlink doesn't work uniformly on Windows.")
upgrade_dir = os.path.join(self.get_temp_dir(), "foo")
other_dir = os.path.join(self.get_temp_dir(), "bar")
os.mkdir(upgrade_dir)
os.mkdir(other_dir)
file_c = os.path.join(other_dir, "c.py")
file_d = os.path.join(upgrade_dir, "d.py")
with open(file_c, "a") as f:
f.write("import foo as f")
os.symlink(file_c, file_d)
upgrader = ast_edits.ASTCodeUpgrader(RenameImports())
upgrader.process_tree_inplace(upgrade_dir)
self.assertTrue(os.path.islink(file_d))
self.assertEqual(file_c, os.readlink(file_d))
# File pointed to by symlink is in a different directory.
# Therefore, it should not be upgraded.
with open(file_c, "r") as f:
self.assertEqual("import foo as f", f.read())
def testUpgradeCopyWithSymlink(self):
if os.name == "nt":
self.skipTest("os.symlink doesn't work uniformly on Windows.")
upgrade_dir = os.path.join(self.get_temp_dir(), "foo")
output_dir = os.path.join(self.get_temp_dir(), "bar")
os.mkdir(upgrade_dir)
file_a = os.path.join(upgrade_dir, "a.py")
file_b = os.path.join(upgrade_dir, "b.py")
with open(file_a, "a") as f:
f.write("import foo as f")
os.symlink(file_a, file_b)
upgrader = ast_edits.ASTCodeUpgrader(RenameImports())
upgrader.process_tree(upgrade_dir, output_dir, copy_other_files=True)
new_file_a = os.path.join(output_dir, "a.py")
new_file_b = os.path.join(output_dir, "b.py")
self.assertTrue(os.path.islink(new_file_b))
self.assertEqual(new_file_a, os.readlink(new_file_b))
with open(new_file_a, "r") as f:
self.assertEqual("import bar as f", f.read())
def testUpgradeCopyWithSymlinkInDifferentDir(self):
if os.name == "nt":
self.skipTest("os.symlink doesn't work uniformly on Windows.")
upgrade_dir = os.path.join(self.get_temp_dir(), "foo")
other_dir = os.path.join(self.get_temp_dir(), "bar")
output_dir = os.path.join(self.get_temp_dir(), "baz")
os.mkdir(upgrade_dir)
os.mkdir(other_dir)
file_a = os.path.join(other_dir, "a.py")
file_b = os.path.join(upgrade_dir, "b.py")
with open(file_a, "a") as f:
f.write("import foo as f")
os.symlink(file_a, file_b)
upgrader = ast_edits.ASTCodeUpgrader(RenameImports())
upgrader.process_tree(upgrade_dir, output_dir, copy_other_files=True)
new_file_b = os.path.join(output_dir, "b.py")
self.assertTrue(os.path.islink(new_file_b))
self.assertEqual(file_a, os.readlink(new_file_b))
with open(file_a, "r") as f:
self.assertEqual("import foo as f", f.read())
if __name__ == "__main__":
test_lib.main()
tensorflow/tools/compatibility/ipynb.py
+170
View File
@@ -0,0 +1,170 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================
"""A module to support operations on ipynb files"""
import collections
import copy
import json
import re
import shutil
import tempfile
CodeLine = collections.namedtuple("CodeLine", ["cell_number", "code"])
def is_python(cell):
"""Checks if the cell consists of Python code."""
return (cell["cell_type"] == "code" # code cells only
and cell["source"] # non-empty cells
and not cell["source"][0].startswith("%%")) # multiline eg: %%bash
def process_file(in_filename, out_filename, upgrader):
"""The function where we inject the support for ipynb upgrade."""
print("Extracting code lines from original notebook")
raw_code, notebook = _get_code(in_filename)
raw_lines = [cl.code for cl in raw_code]
# The function follows the original flow from `upgrader.process_fil`
with tempfile.NamedTemporaryFile("w", delete=False) as temp_file:
processed_file, new_file_content, log, process_errors = (
upgrader.update_string_pasta("\n".join(raw_lines), in_filename))
if temp_file and processed_file:
new_notebook = _update_notebook(notebook, raw_code,
new_file_content.split("\n"))
json.dump(new_notebook, temp_file)
else:
raise SyntaxError(
"Was not able to process the file: \n%s\n" % "".join(log))
files_processed = processed_file
report_text = upgrader._format_log(log, in_filename, out_filename)
errors = process_errors
shutil.move(temp_file.name, out_filename)
return files_processed, report_text, errors
def skip_magic(code_line, magic_list):
"""Checks if the cell has magic, that is not Python-based.
Args:
code_line: A line of Python code
magic_list: A list of jupyter "magic" exceptions
Returns:
If the line jupyter "magic" line, not Python line
>>> skip_magic('!ls -laF', ['%', '!', '?'])
True
"""
for magic in magic_list:
if code_line.startswith(magic):
return True
return False
def check_line_split(code_line):
r"""Checks if a line was split with `\`.
Args:
code_line: A line of Python code
Returns:
If the line was split with `\`
>>> skip_magic("!gcloud ml-engine models create ${MODEL} \\\n")
True
"""
return re.search(r"\\\s*\n$", code_line)
def _get_code(input_file):
"""Loads the ipynb file and returns a list of CodeLines."""
raw_code = []
with open(input_file) as in_file:
notebook = json.load(in_file)
cell_index = 0
for cell in notebook["cells"]:
if is_python(cell):
cell_lines = cell["source"]
is_line_split = False
for line_idx, code_line in enumerate(cell_lines):
# Sometimes, jupyter has more than python code
# Idea is to comment these lines, for upgrade time
if skip_magic(code_line, ["%", "!", "?"]) or is_line_split:
# Found a special character, need to "encode"
code_line = "###!!!" + code_line
# if this cell ends with `\` -> skip the next line
is_line_split = check_line_split(code_line)
if is_line_split:
is_line_split = check_line_split(code_line)
# Sometimes, people leave \n at the end of cell
# in order to migrate only related things, and make the diff
# the smallest -> here is another hack
if (line_idx == len(cell_lines) - 1) and code_line.endswith("\n"):
code_line = code_line.replace("\n", "###===")
# sometimes a line would start with `\n` and content after
# that's the hack for this
raw_code.append(
CodeLine(cell_index,
code_line.rstrip().replace("\n", "###===")))
cell_index += 1
return raw_code, notebook
def _update_notebook(original_notebook, original_raw_lines, updated_code_lines):
"""Updates notebook, once migration is done."""
new_notebook = copy.deepcopy(original_notebook)
# validate that the number of lines is the same
assert len(original_raw_lines) == len(updated_code_lines), \
("The lengths of input and converted files are not the same: "
"{} vs {}".format(len(original_raw_lines), len(updated_code_lines)))
code_cell_idx = 0
for cell in new_notebook["cells"]:
if not is_python(cell):
continue
applicable_lines = [
idx for idx, code_line in enumerate(original_raw_lines)
if code_line.cell_number == code_cell_idx
]
new_code = [updated_code_lines[idx] for idx in applicable_lines]
cell["source"] = "\n".join(new_code).replace("###!!!", "").replace(
"###===", "\n")
code_cell_idx += 1
return new_notebook
tensorflow/tools/compatibility/module_deprecations_v2.py
+66
View File
@@ -0,0 +1,66 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Module deprecation warnings for TensorFlow 2.0."""
from tensorflow.tools.compatibility import ast_edits
_CONTRIB_WARNING = (
ast_edits.ERROR,
"<function name> cannot be converted automatically. tf.contrib will not"
" be distributed with TensorFlow 2.0, please consider an alternative in"
" non-contrib TensorFlow, a community-maintained repository such as "
"tensorflow/addons, or fork the required code.")
_FLAGS_WARNING = (
ast_edits.ERROR,
"tf.flags and tf.app.flags have been removed, please use the argparse or "
"absl modules if you need command line parsing.")
_CONTRIB_CUDNN_RNN_WARNING = (
ast_edits.WARNING,
"(Manual edit required) tf.contrib.cudnn_rnn.* has been deprecated, "
"and the CuDNN kernel has been integrated with "
"tf.keras.layers.LSTM/GRU in TensorFlow 2.0. Please check the new API "
"and use that instead."
)
_CONTRIB_RNN_WARNING = (
ast_edits.WARNING,
"(Manual edit required) tf.contrib.rnn.* has been deprecated, and "
"widely used cells/functions will be moved to tensorflow/addons "
"repository. Please check it there and file Github issues if necessary."
)
_CONTRIB_DIST_STRAT_WARNING = (
ast_edits.WARNING,
"(Manual edit required) tf.contrib.distribute.* have been migrated to "
"tf.distribute.*. Please check out the new module for updated APIs.")
_CONTRIB_SEQ2SEQ_WARNING = (
ast_edits.WARNING,
"(Manual edit required) tf.contrib.seq2seq.* have been migrated to "
"`tfa.seq2seq.*` in TensorFlow Addons. Please see "
"https://github.com/tensorflow/addons for more info.")
MODULE_DEPRECATIONS = {
"tf.contrib": _CONTRIB_WARNING,
"tf.contrib.cudnn_rnn": _CONTRIB_CUDNN_RNN_WARNING,
"tf.contrib.rnn": _CONTRIB_RNN_WARNING,
"tf.flags": _FLAGS_WARNING,
"tf.app.flags": _FLAGS_WARNING,
"tf.contrib.distribute": _CONTRIB_DIST_STRAT_WARNING,
"tf.contrib.seq2seq": _CONTRIB_SEQ2SEQ_WARNING
}
tensorflow/tools/compatibility/renames_v2.py
+1511
View File
File diff suppressed because it is too large Load Diff
tensorflow/tools/compatibility/reorders_v2.py
+136
View File
@@ -0,0 +1,136 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# pylint: disable=line-too-long
"""List of renames to apply when converting from TF 1.0 to TF 2.0.
THIS FILE IS AUTOGENERATED: To update, please run:
bazel run tensorflow/tools/compatibility/update:generate_v2_reorders_map
This file should be updated whenever a function is added to
self.reordered_function_names in tf_upgrade_v2.py.
"""
reorders = {
'tf.argmax': [None, None, 'name', 'dimension', 'output_type'],
'tf.argmin': [None, None, 'name', 'dimension', 'output_type'],
'tf.batch_to_space': [None, 'crops', 'block_size', 'name', 'block_shape'],
'tf.boolean_mask': [None, None, 'name', 'axis'],
'tf.cond': [None, None, None, 'strict', 'name', 'fn1', 'fn2'],
'tf.confusion_matrix': [None, None, None, 'dtype', 'name', 'weights'],
'tf.convert_to_tensor': [None, None, 'name', 'preferred_dtype', 'dtype_hint'],
'tf.data.experimental.RaggedTensorStructure': ['dtype', 'shape', 'ragged_rank'],
'tf.data.experimental.SparseTensorStructure': ['dtype', 'shape'],
'tf.data.experimental.TensorArrayStructure': ['dtype', 'element_shape', 'dynamic_size', 'infer_shape'],
'tf.data.experimental.TensorStructure': ['dtype', 'shape'],
'tf.debugging.assert_all_finite': ['t', 'msg', 'name', 'x', 'message'],
'tf.decode_csv': [None, None, None, None, 'name', 'na_value', 'select_cols'],
'tf.depth_to_space': [None, None, 'name', 'data_format'],
'tf.feature_column.categorical_column_with_vocabulary_file': [None, None, None, 'num_oov_buckets', 'default_value', 'dtype'],
'tf.gather_nd': [None, None, 'name', 'batch_dims'],
'tf.gradients': [None, None, None, None, 'colocate_gradients_with_ops', 'gate_gradients', 'aggregation_method', 'stop_gradients', 'unconnected_gradients'],
'tf.hessians': [None, None, 'name', 'colocate_gradients_with_ops', 'gate_gradients', 'aggregation_method'],
'tf.image.sample_distorted_bounding_box': [None, None, None, 'seed2', 'min_object_covered', 'aspect_ratio_range', 'area_range', 'max_attempts', 'use_image_if_no_bounding_boxes', 'name'],
'tf.initializers.uniform_unit_scaling': ['factor', 'seed', 'dtype'],
'tf.io.decode_csv': [None, None, None, None, 'name', 'na_value', 'select_cols'],
'tf.io.parse_example': [None, None, 'name', 'example_names'],
'tf.io.parse_single_example': [None, None, 'name', 'example_names'],
'tf.io.serialize_many_sparse': [None, 'name', 'out_type'],
'tf.io.serialize_sparse': [None, 'name', 'out_type'],
'tf.linalg.norm': [None, None, None, None, None, 'keep_dims'],
'tf.manip.gather_nd': [None, None, 'name', 'batch_dims'],
'tf.math.argmax': [None, None, 'name', 'dimension', 'output_type'],
'tf.math.argmin': [None, None, 'name', 'dimension', 'output_type'],
'tf.math.confusion_matrix': [None, None, None, 'dtype', 'name', 'weights'],
'tf.math.in_top_k': ['predictions', 'targets', 'k', 'name'],
'tf.math.reduce_all': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.math.reduce_any': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.math.reduce_logsumexp': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.math.reduce_max': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.math.reduce_mean': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.math.reduce_min': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.math.reduce_prod': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.math.reduce_sum': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.multinomial': [None, None, 'seed', 'name', 'output_dtype'],
'tf.nn.avg_pool': ['value', 'ksize', 'strides', 'padding', 'data_format', 'name', 'input'],
'tf.nn.avg_pool2d': ['value', 'ksize', 'strides', 'padding', 'data_format', 'name', 'input'],
'tf.nn.conv1d': ['value', 'filters', 'stride', 'padding', 'use_cudnn_on_gpu', 'data_format', 'name', 'input', 'dilations'],
'tf.nn.conv2d': [None, 'filter', 'strides', 'padding', 'use_cudnn_on_gpu', 'data_format', 'dilations', 'name', 'filters'],
'tf.nn.conv2d_backprop_input': ['input_sizes', 'filter', 'out_backprop', 'strides', 'padding', 'use_cudnn_on_gpu', 'data_format', 'dilations', 'name', 'filters'],
'tf.nn.convolution': [None, 'filter', 'padding', 'strides', 'dilation_rate', 'name', 'data_format', 'filters', 'dilations'],
'tf.nn.crelu': [None, 'name', 'axis'],
'tf.nn.ctc_beam_search_decoder': ['inputs', 'sequence_length', 'beam_width', 'top_paths', 'merge_repeated'],
'tf.nn.depth_to_space': [None, None, 'name', 'data_format'],
'tf.nn.depthwise_conv2d': [None, None, None, None, 'rate', 'name', 'data_format', 'dilations'],
'tf.nn.embedding_lookup': [None, None, 'partition_strategy', 'name', 'validate_indices', 'max_norm'],
'tf.nn.embedding_lookup_sparse': [None, None, None, 'partition_strategy', 'name', 'combiner', 'max_norm', 'allow_fast_lookup'],
'tf.nn.fractional_avg_pool': ['value', 'pooling_ratio', 'pseudo_random', 'overlapping', 'deterministic', 'seed', 'seed2', 'name'],
'tf.nn.fractional_max_pool': ['value', 'pooling_ratio', 'pseudo_random', 'overlapping', 'deterministic', 'seed', 'seed2', 'name'],
'tf.nn.in_top_k': ['predictions', 'targets', 'k', 'name'],
'tf.nn.max_pool': ['value', 'ksize', 'strides', 'padding', 'data_format', 'name', 'input'],
'tf.nn.moments': [None, None, None, 'name', 'keep_dims', 'keepdims'],
'tf.nn.pool': [None, None, None, 'padding', 'dilation_rate', 'strides', 'name', 'data_format', 'dilations'],
'tf.nn.separable_conv2d': [None, None, None, None, None, 'rate', 'name', 'data_format', 'dilations'],
'tf.nn.softmax_cross_entropy_with_logits': ['labels', 'logits', 'dim', 'name', 'axis'],
'tf.nn.space_to_batch': [None, 'paddings', 'block_size', 'name', 'block_shape'],
'tf.nn.space_to_depth': [None, None, 'name', 'data_format'],
'tf.nn.weighted_moments': [None, None, None, 'name', 'keep_dims', 'keepdims'],
'tf.norm': [None, None, None, None, None, 'keep_dims'],
'tf.pad': [None, None, None, 'name', 'constant_values'],
'tf.parse_example': [None, None, 'name', 'example_names'],
'tf.parse_single_example': [None, None, 'name', 'example_names'],
'tf.quantize_v2': [None, None, None, None, None, 'name', 'round_mode', 'narrow_range', 'axis', 'ensure_minimum_range'],
'tf.random.multinomial': [None, None, 'seed', 'name', 'output_dtype'],
'tf.random.poisson': ['lam', 'shape', 'dtype', 'seed', 'name'],
'tf.random_poisson': ['lam', 'shape', 'dtype', 'seed', 'name'],
'tf.reduce_all': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.reduce_any': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.reduce_join': [None, None, 'keep_dims', 'separator', 'name', 'reduction_indices', 'keepdims'],
'tf.reduce_logsumexp': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.reduce_max': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.reduce_mean': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.reduce_min': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.reduce_prod': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.reduce_sum': [None, None, None, None, 'reduction_indices', 'keep_dims'],
'tf.reverse_sequence': [None, None, None, None, None, 'seq_dim', 'batch_dim'],
'tf.serialize_many_sparse': [None, 'name', 'out_type'],
'tf.serialize_sparse': [None, 'name', 'out_type'],
'tf.shape': [None, 'name', 'out_type'],
'tf.size': [None, 'name', 'out_type'],
'tf.space_to_batch': [None, 'paddings', 'block_size', 'name', 'block_shape'],
'tf.space_to_depth': [None, None, 'name', 'data_format'],
'tf.sparse.add': [None, None, None, 'thresh'],
'tf.sparse.concat': [None, None, 'name', 'expand_nonconcat_dim', 'concat_dim', 'expand_nonconcat_dims'],
'tf.sparse.reduce_max': [None, None, None, 'reduction_axes', 'keep_dims'],
'tf.sparse.segment_mean': [None, None, None, 'name', 'num_segments', 'sparse_gradient'],
'tf.sparse.segment_sqrt_n': [None, None, None, 'name', 'num_segments', 'sparse_gradient'],
'tf.sparse.segment_sum': [None, None, None, 'name', 'num_segments', 'sparse_gradient'],
'tf.sparse.split': ['keyword_required', 'sp_input', 'num_split', 'axis', 'name', 'split_dim'],
'tf.sparse_add': [None, None, None, 'thresh'],
'tf.sparse_concat': [None, None, 'name', 'expand_nonconcat_dim', 'concat_dim', 'expand_nonconcat_dims'],
'tf.sparse_matmul': [None, None, None, None, 'a_is_sparse', 'b_is_sparse', 'name'],
'tf.sparse_reduce_max': [None, None, None, 'reduction_axes', 'keep_dims'],
'tf.sparse_segment_mean': [None, None, None, 'name', 'num_segments', 'sparse_gradient'],
'tf.sparse_segment_sqrt_n': [None, None, None, 'name', 'num_segments', 'sparse_gradient'],
'tf.sparse_segment_sum': [None, None, None, 'name', 'num_segments', 'sparse_gradient'],
'tf.sparse_split': ['keyword_required', 'sp_input', 'num_split', 'axis', 'name', 'split_dim'],
'tf.strings.length': [None, 'name', 'unit'],
'tf.strings.reduce_join': [None, None, 'keep_dims', 'separator', 'name', 'reduction_indices', 'keepdims'],
'tf.strings.substr': [None, None, None, 'name', 'unit'],
'tf.substr': [None, None, None, 'name', 'unit'],
'tf.test.assert_equal_graph_def': ['actual', 'expected', 'checkpoint_v2', 'hash_table_shared_name'],
'tf.transpose': [None, None, 'name', 'conjugate'],
'tf.tuple': [None, 'name', 'control_inputs'],
'tf.uniform_unit_scaling_initializer': ['factor', 'seed', 'dtype'],
'tf.verify_tensor_all_finite': ['t', 'msg', 'name', 'x', 'message'],
'tf.while_loop': ['cond', 'body', 'loop_vars', 'shape_invariants', 'parallel_iterations', 'back_prop', 'swap_memory', 'name', 'maximum_iterations', 'return_same_structure']
}
tensorflow/tools/compatibility/testdata/test_file_v0_11.py
Vendored
+232
View File
@@ -0,0 +1,232 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tf upgrader."""
import shutil
import tempfile
import numpy as np
import tensorflow as tf
from tensorflow.python.framework import test_util
from tensorflow.python.platform import test as test_lib
class TestUpgrade(test_util.TensorFlowTestCase):
"""Test various APIs that have been changed in 1.0.
This test will not run in current TensorFlow, but did run in 0.11.
This file is intended to be converted by a genrule() that uses the converter
so that a 1.0 compatible version of this file is generated. That is run as
a unit test if the converter is successful.
"""
@test_util.run_v1_only("b/120545219")
def testArgRenames(self):
with self.cached_session():
a = [[1., 2., 3.], [4., 5., 6.]]
b = [[True, False, False], [False, True, True]]
dim0 = [1]
dim1 = [1]
self.assertAllEqual(
tf.reduce_any(
b, reduction_indices=dim0).eval(), [True, True])
self.assertAllEqual(
tf.reduce_all(
b, reduction_indices=[0]).eval(), [False, False, False])
self.assertAllEqual(
tf.reduce_all(
b, reduction_indices=dim1).eval(), [False, False])
self.assertAllEqual(
tf.reduce_sum(
a, reduction_indices=[1]).eval(), [6., 15.])
self.assertAllEqual(
tf.reduce_sum(
a, reduction_indices=[0, 1]).eval(), 21.0)
self.assertAllEqual(tf.reduce_sum(a, [0, 1]).eval(), 21.0)
self.assertAllEqual(
tf.reduce_prod(
a, reduction_indices=[1]).eval(), [6., 120.])
self.assertAllEqual(
tf.reduce_prod(
a, reduction_indices=[0, 1]).eval(), 720.0)
self.assertAllEqual(tf.reduce_prod(a, [0, 1]).eval(), 720.0)
self.assertAllEqual(
tf.reduce_mean(
a, reduction_indices=[1]).eval(), [2., 5.])
self.assertAllEqual(
tf.reduce_mean(
a, reduction_indices=[0, 1]).eval(), 3.5)
self.assertAllEqual(tf.reduce_mean(a, [0, 1]).eval(), 3.5)
self.assertAllEqual(
tf.reduce_min(
a, reduction_indices=[1]).eval(), [1., 4.])
self.assertAllEqual(
tf.reduce_min(
a, reduction_indices=[0, 1]).eval(), 1.0)
self.assertAllEqual(tf.reduce_min(a, [0, 1]).eval(), 1.0)
self.assertAllEqual(
tf.reduce_max(
a, reduction_indices=[1]).eval(), [3., 6.])
self.assertAllEqual(
tf.reduce_max(
a, reduction_indices=[0, 1]).eval(), 6.0)
self.assertAllEqual(tf.reduce_max(a, [0, 1]).eval(), 6.0)
self.assertAllClose(tf.reduce_logsumexp(a, reduction_indices=[1]).eval(),
[3.40760589, 6.40760612])
self.assertAllClose(
tf.reduce_logsumexp(a, reduction_indices=[0, 1]).eval(),
6.45619344711)
self.assertAllClose(
tf.reduce_logsumexp(a, [0, 1]).eval(), 6.45619344711)
self.assertAllEqual(
tf.expand_dims([[1, 2], [3, 4]], axis=1).eval(),
[[[1, 2]], [[3, 4]]])
@test_util.run_v1_only("b/120545219")
def testArgMinMax(self):
with self.cached_session():
self.assertAllEqual(
tf.argmin([[1, 2, 3], [4, 1, 0]], dimension=1).eval(),
[0, 2])
self.assertAllEqual(
tf.argmin([[1, 2, 3], [4, 1, 0]], dimension=0).eval(),
[0, 1, 1])
self.assertAllEqual(
tf.argmax([[1, 2, 3], [4, 1, 0]], dimension=1).eval(),
[2, 0])
self.assertAllEqual(
tf.argmax([[1, 2, 3], [4, 1, 0]], dimension=0).eval(),
[1, 0, 0])
@test_util.run_v1_only("b/120545219")
def testExpandAndSqueeze(self):
with self.cached_session():
# TODO(aselle): sparse_split, sparse_reduce_sum,
# sparse_reduce_sum_sparse, reduce_join
a = [[1, 2, 3]]
self.assertAllEqual(tf.expand_dims(tf.squeeze(a, [0]), 0).eval(),
a)
self.assertAllEqual(tf.squeeze(tf.expand_dims(a, 1), [1]).eval(),
a)
self.assertAllEqual(
tf.expand_dims(tf.squeeze([[1, 2, 3]], axis=[0]), dim=0).eval(), a)
self.assertAllEqual(
tf.squeeze(tf.expand_dims([[1, 2, 3]], dim=1), axis=[1]).eval(), a)
self.assertAllEqual(
tf.squeeze(tf.expand_dims([[1, 2, 3]], dim=1), axis=[1]).eval(), a)
@test_util.run_v1_only("b/120545219")
def testArithmeticRenames(self):
with self.cached_session() as s:
stuff = tf.split(1, 2, [[1, 2, 3, 4], [4, 5, 6, 7]])
vals = s.run(stuff)
self.assertAllEqual(vals,
[[[1, 2], [4, 5]], [[3, 4], [6, 7]]])
self.assertAllEqual(
tf.neg(tf.mul(tf.add(1, 2), tf.sub(5, 3))).eval(),
-6)
self.assertAllEqual(
s.run(tf.listdiff([1, 2, 3], [3, 3, 4]))[0], [1, 2])
self.assertAllEqual(
tf.list_diff([1, 2, 3], [3, 3, 4])[0].eval(), [1, 2])
a = [[1., 2., 3.], [4., 5., 6.]]
foo = np.where(np.less(a, 2), np.negative(a), a)
self.assertAllEqual(
tf.select(tf.less(a, 2), tf.neg(a), a).eval(),
foo)
self.assertAllEqual(
tf.complex_abs(tf.constant(3 + 4.j)).eval(),
5)
# # TODO(aselle): (tf.batch_*)
# ]
@test_util.run_v1_only("b/120545219")
def testBatchAndSvd(self):
with self.cached_session():
mat = [[1., 2.], [2., 3.]]
batched_mat = tf.expand_dims(mat, [0])
result = tf.matmul(mat, mat).eval()
result_batched = tf.batch_matmul(batched_mat, batched_mat).eval()
self.assertAllEqual(result_batched, np.expand_dims(result, 0))
self.assertAllEqual(
tf.svd(mat, False, True).eval(),
tf.svd(mat, compute_uv=False, full_matrices=True).eval())
@test_util.run_v1_only("b/120545219")
def testCrossEntropy(self):
# TODO(aselle): Test sparse_softmax_...
with self.cached_session():
labels = [.8, .5, .2, .1]
logits = [.9, .1, .3, .1]
self.assertAllEqual(
tf.nn.softmax_cross_entropy_with_logits(
logits, labels).eval(),
tf.nn.softmax_cross_entropy_with_logits(
labels=labels, logits=logits).eval())
self.assertAllEqual(
tf.nn.sigmoid_cross_entropy_with_logits(
logits, labels).eval(),
tf.nn.sigmoid_cross_entropy_with_logits(
labels=labels, logits=logits).eval())
@test_util.run_v1_only("b/120545219")
def testVariables(self):
with self.cached_session() as s:
# make some variables
_ = [tf.Variable([1, 2, 3], dtype=tf.float32),
tf.Variable([1, 2, 3], dtype=tf.int32)]
s.run(tf.global_variables_initializer())
_ = [v.name for v in tf.all_variables()]
_ = [v.name for v in tf.local_variables()]
@test_util.run_v1_only("b/120545219")
def testSummaries(self):
with self.cached_session() as s:
var = tf.Variable([1, 2, 3], dtype=tf.float32)
s.run(tf.global_variables_initializer())
x, y = np.meshgrid(np.linspace(-10, 10, 256), np.linspace(-10, 10, 256))
image = np.sin(x**2 + y**2) / np.sqrt(x**2 + y**2) * .5 + .5
image = image[None, :, :, None]
# make a dummy sound
freq = 440 # A = 440Hz
sampling_frequency = 11000
audio = np.sin(2 * np.pi * np.linspace(0, 1, sampling_frequency) * freq)
audio = audio[None, :, None]
test_dir = tempfile.mkdtemp()
# test summaries
writer = tf.train.SummaryWriter(test_dir)
summaries = [
tf.scalar_summary("scalar_var", var[0]),
tf.scalar_summary("scalar_reduce_var", tf.reduce_sum(var)),
tf.histogram_summary("var_histogram", var),
tf.image_summary("sin_image", image),
tf.audio_summary("sin_wave", audio, sampling_frequency),
]
run_summaries = s.run(summaries)
writer.add_summary(s.run(tf.merge_summary(inputs=run_summaries)))
# This is redundant, but we want to be able to rewrite the command
writer.add_summary(s.run(tf.merge_all_summaries()))
writer.close()
shutil.rmtree(test_dir)
if __name__ == "__main__":
test_lib.main()
tensorflow/tools/compatibility/testdata/test_file_v1_12.py
Vendored
+81
View File
@@ -0,0 +1,81 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tf upgrader."""
import tensorflow.compat.v1 as tf
from tensorflow.python.framework import test_util
from tensorflow.python.platform import test as test_lib
_TEST_VERSION = 1
class TestUpgrade(test_util.TensorFlowTestCase):
"""Test various APIs that have been changed in 2.0."""
@classmethod
def setUpClass(cls):
cls._tf_api_version = 1 if hasattr(tf, 'contrib') else 2
def setUp(self):
tf.compat.v1.enable_v2_behavior()
def testRenames(self):
self.assertAllClose(1.04719755, tf.acos(0.5))
self.assertAllClose(0.5, tf.rsqrt(4.0))
def testSerializeSparseTensor(self):
sp_input = tf.SparseTensor(
indices=tf.constant([[1]], dtype=tf.int64),
values=tf.constant([2], dtype=tf.int64),
dense_shape=[2])
with self.cached_session():
serialized_sp = tf.serialize_sparse(sp_input, 'serialize_name', tf.string)
self.assertEqual((3,), serialized_sp.shape)
self.assertTrue(serialized_sp[0].numpy()) # check non-empty
def testSerializeManySparse(self):
sp_input = tf.SparseTensor(
indices=tf.constant([[0, 1]], dtype=tf.int64),
values=tf.constant([2], dtype=tf.int64),
dense_shape=[1, 2])
with self.cached_session():
serialized_sp = tf.serialize_many_sparse(
sp_input, 'serialize_name', tf.string)
self.assertEqual((1, 3), serialized_sp.shape)
def testArgMaxMin(self):
self.assertAllClose(
[1],
tf.argmax([[1, 3, 2]], name='abc', dimension=1))
self.assertAllClose(
[0, 0, 0],
tf.argmax([[1, 3, 2]], dimension=0))
self.assertAllClose(
[0],
tf.argmin([[1, 3, 2]], name='abc', dimension=1))
def testSoftmaxCrossEntropyWithLogits(self):
out = tf.nn.softmax_cross_entropy_with_logits(
logits=[0.1, 0.8], labels=[0, 1])
self.assertAllClose(out, 0.40318608)
out = tf.nn.softmax_cross_entropy_with_logits_v2(
logits=[0.1, 0.8], labels=[0, 1])
self.assertAllClose(out, 0.40318608)
if __name__ == "__main__":
test_lib.main()
tensorflow/tools/compatibility/tf_upgrade.py
+251
View File
@@ -0,0 +1,251 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Upgrader for Python scripts from pre-1.0 TensorFlow to 1.0 TensorFlow."""
import argparse
from tensorflow.tools.compatibility import ast_edits
class TFAPIChangeSpec(ast_edits.APIChangeSpec):
"""List of maps that describe what changed in the API."""
def __init__(self):
# Maps from a function name to a dictionary that describes how to
# map from an old argument keyword to the new argument keyword.
self.function_keyword_renames = {
"tf.batch_matmul": {
"adj_x": "adjoint_a",
"adj_y": "adjoint_b",
},
"tf.count_nonzero": {
"reduction_indices": "axis"
},
"tf.reduce_all": {
"reduction_indices": "axis"
},
"tf.reduce_any": {
"reduction_indices": "axis"
},
"tf.reduce_max": {
"reduction_indices": "axis"
},
"tf.reduce_mean": {
"reduction_indices": "axis"
},
"tf.reduce_min": {
"reduction_indices": "axis"
},
"tf.reduce_prod": {
"reduction_indices": "axis"
},
"tf.reduce_sum": {
"reduction_indices": "axis"
},
"tf.reduce_logsumexp": {
"reduction_indices": "axis"
},
"tf.expand_dims": {
"dim": "axis"
},
"tf.argmax": {
"dimension": "axis"
},
"tf.argmin": {
"dimension": "axis"
},
"tf.reduce_join": {
"reduction_indices": "axis"
},
"tf.sparse_concat": {
"concat_dim": "axis"
},
"tf.sparse_split": {
"split_dim": "axis"
},
"tf.sparse_reduce_sum": {
"reduction_axes": "axis"
},
"tf.reverse_sequence": {
"seq_dim": "seq_axis",
"batch_dim": "batch_axis"
},
"tf.sparse_reduce_sum_sparse": {
"reduction_axes": "axis"
},
"tf.squeeze": {
"squeeze_dims": "axis"
},
"tf.split": {
"split_dim": "axis",
"num_split": "num_or_size_splits"
},
"tf.concat": {
"concat_dim": "axis"
},
}
# Mapping from function to the new name of the function
self.symbol_renames = {
"tf.inv": "tf.reciprocal",
"tf.contrib.deprecated.scalar_summary": "tf.summary.scalar",
"tf.contrib.deprecated.histogram_summary": "tf.summary.histogram",
"tf.listdiff": "tf.setdiff1d",
"tf.list_diff": "tf.setdiff1d",
"tf.mul": "tf.multiply",
"tf.neg": "tf.negative",
"tf.sub": "tf.subtract",
"tf.train.SummaryWriter": "tf.summary.FileWriter",
"tf.scalar_summary": "tf.summary.scalar",
"tf.histogram_summary": "tf.summary.histogram",
"tf.audio_summary": "tf.summary.audio",
"tf.image_summary": "tf.summary.image",
"tf.merge_summary": "tf.summary.merge",
"tf.merge_all_summaries": "tf.summary.merge_all",
"tf.image.per_image_whitening": "tf.image.per_image_standardization",
"tf.all_variables": "tf.global_variables",
"tf.VARIABLES": "tf.GLOBAL_VARIABLES",
"tf.initialize_all_variables": "tf.global_variables_initializer",
"tf.initialize_variables": "tf.variables_initializer",
"tf.initialize_local_variables": "tf.local_variables_initializer",
"tf.batch_matrix_diag": "tf.matrix_diag",
"tf.batch_band_part": "tf.band_part",
"tf.batch_set_diag": "tf.set_diag",
"tf.batch_matrix_transpose": "tf.matrix_transpose",
"tf.batch_matrix_determinant": "tf.matrix_determinant",
"tf.batch_matrix_inverse": "tf.matrix_inverse",
"tf.batch_cholesky": "tf.cholesky",
"tf.batch_cholesky_solve": "tf.cholesky_solve",
"tf.batch_matrix_solve": "tf.matrix_solve",
"tf.batch_matrix_triangular_solve": "tf.matrix_triangular_solve",
"tf.batch_matrix_solve_ls": "tf.matrix_solve_ls",
"tf.batch_self_adjoint_eig": "tf.self_adjoint_eig",
"tf.batch_self_adjoint_eigvals": "tf.self_adjoint_eigvals",
"tf.batch_svd": "tf.svd",
"tf.batch_fft": "tf.fft",
"tf.batch_ifft": "tf.ifft",
"tf.batch_fft2d": "tf.fft2d",
"tf.batch_ifft2d": "tf.ifft2d",
"tf.batch_fft3d": "tf.fft3d",
"tf.batch_ifft3d": "tf.ifft3d",
"tf.select": "tf.where",
"tf.complex_abs": "tf.abs",
"tf.batch_matmul": "tf.matmul",
"tf.pack": "tf.stack",
"tf.unpack": "tf.unstack",
"tf.op_scope": "tf.name_scope",
}
self.change_to_function = {
"tf.ones_initializer",
"tf.zeros_initializer",
}
# Functions that were reordered should be changed to the new keyword args
# for safety, if positional arguments are used. If you have reversed the
# positional arguments yourself, this could do the wrong thing.
self.function_reorders = {
"tf.split": ["axis", "num_or_size_splits", "value", "name"],
"tf.sparse_split": ["axis", "num_or_size_splits", "value", "name"],
"tf.concat": ["concat_dim", "values", "name"],
"tf.svd": ["tensor", "compute_uv", "full_matrices", "name"],
"tf.nn.softmax_cross_entropy_with_logits": [
"logits", "labels", "dim", "name"
],
"tf.nn.sparse_softmax_cross_entropy_with_logits": [
"logits", "labels", "name"
],
"tf.nn.sigmoid_cross_entropy_with_logits": ["logits", "labels", "name"],
"tf.op_scope": ["values", "name", "default_name"],
}
# Warnings that should be printed if corresponding functions are used.
self.function_warnings = {
"tf.reverse": (
ast_edits.ERROR,
"tf.reverse has had its argument semantics changed "
"significantly. The converter cannot detect this reliably, so "
"you need to inspect this usage manually.\n"),
}
self.module_deprecations = {}
if __name__ == "__main__":
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""Convert a TensorFlow Python file to 1.0
Simple usage:
tf_convert.py --infile foo.py --outfile bar.py
tf_convert.py --intree ~/code/old --outtree ~/code/new
""")
parser.add_argument(
"--infile",
dest="input_file",
help="If converting a single file, the name of the file "
"to convert")
parser.add_argument(
"--outfile",
dest="output_file",
help="If converting a single file, the output filename.")
parser.add_argument(
"--intree",
dest="input_tree",
help="If converting a whole tree of files, the directory "
"to read from (relative or absolute).")
parser.add_argument(
"--outtree",
dest="output_tree",
help="If converting a whole tree of files, the output "
"directory (relative or absolute).")
parser.add_argument(
"--copyotherfiles",
dest="copy_other_files",
help=("If converting a whole tree of files, whether to "
"copy the other files."),
type=bool,
default=False)
parser.add_argument(
"--reportfile",
dest="report_filename",
help=("The name of the file where the report log is "
"stored."
"(default: %(default)s)"),
default="report.txt")
args = parser.parse_args()
upgrade = ast_edits.ASTCodeUpgrader(TFAPIChangeSpec())
report_text = None
report_filename = args.report_filename
files_processed = 0
if args.input_file:
files_processed, report_text, errors = upgrade.process_file(
args.input_file, args.output_file)
files_processed = 1
elif args.input_tree:
files_processed, report_text, errors = upgrade.process_tree(
args.input_tree, args.output_tree, args.copy_other_files)
else:
parser.print_help()
if report_text:
open(report_filename, "w").write(report_text)
print("TensorFlow 1.0 Upgrade Script")
print("-----------------------------")
print("Converted %d files\n" % files_processed)
print("Detected %d errors that require attention" % len(errors))
print("-" * 80)
print("\n".join(errors))
print("\nMake sure to read the detailed log %r\n" % report_filename)
tensorflow/tools/compatibility/tf_upgrade_test.py
+148
View File
@@ -0,0 +1,148 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tf upgrader."""
import io
import os
import tempfile
from tensorflow.python.framework import test_util
from tensorflow.python.platform import test as test_lib
from tensorflow.tools.compatibility import ast_edits
from tensorflow.tools.compatibility import tf_upgrade
class TestUpgrade(test_util.TensorFlowTestCase):
"""Test various APIs that have been changed in 1.0.
We also test whether a converted file is executable. test_file_v0_11.py
aims to exhaustively test that API changes are convertible and actually
work when run with current TensorFlow.
"""
def _upgrade(self, old_file_text):
in_file = io.StringIO(old_file_text)
out_file = io.StringIO()
upgrader = ast_edits.ASTCodeUpgrader(tf_upgrade.TFAPIChangeSpec())
count, report, errors = (
upgrader.process_opened_file("test.py", in_file,
"test_out.py", out_file))
return count, report, errors, out_file.getvalue()
def testParseError(self):
_, report, unused_errors, unused_new_text = self._upgrade(
"import tensorflow as tf\na + \n")
self.assertNotEqual(report.find("Failed to parse"), -1)
def testReport(self):
text = "tf.mul(a, b)\n"
_, report, unused_errors, unused_new_text = self._upgrade(text)
# This is not a complete test, but it is a sanity test that a report
# is generating information.
self.assertTrue(report.find("Renamed function `tf.mul` to `tf.multiply`"))
def testRename(self):
text = "tf.mul(a, tf.sub(b, c))\n"
_, unused_report, unused_errors, new_text = self._upgrade(text)
self.assertEqual(new_text, "tf.multiply(a, tf.subtract(b, c))\n")
def testRenamePack(self):
text = "tf.pack(a)\n"
_, unused_report, unused_errors, new_text = self._upgrade(text)
self.assertEqual(new_text, "tf.stack(a)\n")
text = "tf.unpack(a)\n"
_, unused_report, unused_errors, new_text = self._upgrade(text)
self.assertEqual(new_text, "tf.unstack(a)\n")
def testReorder(self):
text = "tf.concat(a, b)\ntf.split(a, b, c)\n"
_, unused_report, unused_errors, new_text = self._upgrade(text)
self.assertEqual(new_text, "tf.concat(axis=a, values=b)\n"
"tf.split(axis=a, num_or_size_splits=b, value=c)\n")
def testConcatReorderWithKeywordArgs(self):
text = "tf.concat(concat_dim=a, values=b)\n"
_, unused_report, unused_errors, new_text = self._upgrade(text)
self.assertEqual(new_text, "tf.concat(axis=a, values=b)\n")
text = "tf.concat(values=b, concat_dim=a)\n"
_, unused_report, unused_errors, new_text = self._upgrade(text)
self.assertEqual(new_text, "tf.concat(values=b, axis=a)\n")
text = "tf.concat(a, values=b)\n"
_, unused_report, unused_errors, new_text = self._upgrade(text)
self.assertEqual(new_text, "tf.concat(axis=a, values=b)\n")
def testConcatReorderNested(self):
text = "tf.concat(a, tf.concat(c, d))\n"
_, unused_report, unused_errors, new_text = self._upgrade(text)
self.assertEqual(
new_text, "tf.concat(axis=a, values=tf.concat(axis=c, values=d))\n")
def testInitializers(self):
text = ("tf.zeros_initializer;tf.zeros_initializer ()\n"
"tf.ones_initializer;tf.ones_initializer ()\n")
_, unused_report, unused_errors, new_text = self._upgrade(text)
self.assertEqual(
new_text, "tf.zeros_initializer();tf.zeros_initializer ()\n"
"tf.ones_initializer();tf.ones_initializer ()\n")
def testKeyword(self):
text = "tf.reduce_any(a, reduction_indices=[1, 2])\n"
_, unused_report, unused_errors, new_text = self._upgrade(text)
self.assertEqual(new_text, "tf.reduce_any(a, axis=[1, 2])\n")
def testComplexExpression(self):
text = "(foo + bar)[a].word()"
_ = self._upgrade(text)
def testReverse(self):
text = "tf.reverse(a, b)\n"
_, unused_report, errors, new_text = self._upgrade(text)
self.assertEqual(new_text, new_text)
self.assertIn("tf.reverse requires manual check", errors[0])
def testListComprehension(self):
def _test(input, output): # pylint: disable=redefined-builtin
_, unused_report, errors, new_text = self._upgrade(input)
self.assertEqual(new_text, output)
_test("tf.concat(0, \t[x for x in y])\n",
"tf.concat(axis=0, \tvalues=[x for x in y])\n")
_test("tf.concat(0,[x for x in y])\n",
"tf.concat(axis=0,values=[x for x in y])\n")
_test("tf.concat(0,[\nx for x in y])\n",
"tf.concat(axis=0,values=[\nx for x in y])\n")
_test("tf.concat(0,[\n \tx for x in y])\n",
"tf.concat(axis=0,values=[\n \tx for x in y])\n")
# TODO(aselle): Explicitly not testing command line interface and process_tree
# for now, since this is a one off utility.
class TestUpgradeFiles(test_util.TensorFlowTestCase):
def testInplace(self):
"""Check to make sure we don't have a file system race."""
temp_file = tempfile.NamedTemporaryFile("w", delete=False)
original = "tf.mul(a, b)\n"
upgraded = "tf.multiply(a, b)\n"
temp_file.write(original)
temp_file.close()
upgrader = ast_edits.ASTCodeUpgrader(tf_upgrade.TFAPIChangeSpec())
upgrader.process_file(temp_file.name, temp_file.name)
self.assertAllEqual(open(temp_file.name).read(), upgraded)
os.unlink(temp_file.name)
if __name__ == "__main__":
test_lib.main()
tensorflow/tools/compatibility/tf_upgrade_v2.py
+2500
View File
File diff suppressed because it is too large Load Diff
tensorflow/tools/compatibility/tf_upgrade_v2_main.py
+206
View File
@@ -0,0 +1,206 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Upgrader for Python scripts from 1.x TensorFlow to 2.0 TensorFlow."""
import argparse
from tensorflow.tools.compatibility import ast_edits
from tensorflow.tools.compatibility import ipynb
from tensorflow.tools.compatibility import tf_upgrade_v2
from tensorflow.tools.compatibility import tf_upgrade_v2_safety
# Make straightforward changes to convert to 2.0. In harder cases,
# use compat.v1.
_DEFAULT_MODE = "DEFAULT"
# Convert to use compat.v1.
_SAFETY_MODE = "SAFETY"
# Whether to rename to compat.v2
_IMPORT_RENAME_DEFAULT = False
def process_file(in_filename, out_filename, upgrader):
"""Process a file of type `.py` or `.ipynb`."""
if in_filename.endswith(".py"):
files_processed, report_text, errors = \
upgrader.process_file(in_filename, out_filename)
elif in_filename.endswith(".ipynb"):
files_processed, report_text, errors = \
ipynb.process_file(in_filename, out_filename, upgrader)
else:
raise NotImplementedError(
"Currently converter only supports python or ipynb")
return files_processed, report_text, errors
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""Convert a TensorFlow Python file from 1.x to 2.0
Simple usage:
tf_upgrade_v2.py --infile foo.py --outfile bar.py
tf_upgrade_v2.py --infile foo.ipynb --outfile bar.ipynb
tf_upgrade_v2.py --intree ~/code/old --outtree ~/code/new
""")
parser.add_argument(
"--infile",
dest="input_file",
help="If converting a single file, the name of the file "
"to convert")
parser.add_argument(
"--outfile",
dest="output_file",
help="If converting a single file, the output filename.")
parser.add_argument(
"--intree",
dest="input_tree",
help="If converting a whole tree of files, the directory "
"to read from (relative or absolute).")
parser.add_argument(
"--outtree",
dest="output_tree",
help="If converting a whole tree of files, the output "
"directory (relative or absolute).")
parser.add_argument(
"--copyotherfiles",
dest="copy_other_files",
help=("If converting a whole tree of files, whether to "
"copy the other files."),
type=bool,
default=True)
parser.add_argument(
"--inplace",
dest="in_place",
help=("If converting a set of files, whether to "
"allow the conversion to be performed on the "
"input files."),
action="store_true")
parser.add_argument(
"--no_import_rename",
dest="no_import_rename",
help=("Not to rename import to compat.v2 explicitly."),
action="store_true")
parser.add_argument(
"--no_upgrade_compat_v1_import",
dest="no_upgrade_compat_v1_import",
help=("If specified, don't upgrade explicit imports of "
"`tensorflow.compat.v1 as tf` to the v2 APIs. Otherwise, "
"explicit imports of the form `tensorflow.compat.v1 as tf` will "
"be upgraded."),
action="store_true")
parser.add_argument(
"--reportfile",
dest="report_filename",
help=("The name of the file where the report log is "
"stored."
"(default: %(default)s)"),
default="report.txt")
parser.add_argument(
"--mode",
dest="mode",
choices=[_DEFAULT_MODE, _SAFETY_MODE],
help=("Upgrade script mode. Supported modes:\n"
"%s: Perform only straightforward conversions to upgrade to "
"2.0. In more difficult cases, switch to use compat.v1.\n"
"%s: Keep 1.* code intact and import compat.v1 "
"module." %
(_DEFAULT_MODE, _SAFETY_MODE)),
default=_DEFAULT_MODE)
parser.add_argument(
"--print_all",
dest="print_all",
help="Print full log to stdout instead of just printing errors",
action="store_true")
args = parser.parse_args()
if args.mode == _SAFETY_MODE:
change_spec = tf_upgrade_v2_safety.TFAPIChangeSpec()
else:
if args.no_import_rename:
change_spec = tf_upgrade_v2.TFAPIChangeSpec(
import_rename=False,
upgrade_compat_v1_import=not args.no_upgrade_compat_v1_import)
else:
change_spec = tf_upgrade_v2.TFAPIChangeSpec(
import_rename=_IMPORT_RENAME_DEFAULT,
upgrade_compat_v1_import=not args.no_upgrade_compat_v1_import)
upgrade = ast_edits.ASTCodeUpgrader(change_spec)
report_text = None
report_filename = args.report_filename
files_processed = 0
if args.input_file:
if not args.in_place and not args.output_file:
raise ValueError(
"--outfile=<output file> argument is required when converting a "
"single file.")
if args.in_place and args.output_file:
raise ValueError("--outfile argument is invalid when converting in place")
output_file = args.input_file if args.in_place else args.output_file
files_processed, report_text, errors = process_file(
args.input_file, output_file, upgrade)
errors = {args.input_file: errors}
files_processed = 1
elif args.input_tree:
if not args.in_place and not args.output_tree:
raise ValueError(
"--outtree=<output directory> argument is required when converting a "
"file tree.")
if args.in_place and args.output_tree:
raise ValueError("--outtree argument is invalid when converting in place")
output_tree = args.input_tree if args.in_place else args.output_tree
files_processed, report_text, errors = upgrade.process_tree(
args.input_tree, output_tree, args.copy_other_files)
else:
parser.print_help()
if report_text:
num_errors = 0
report = []
for f in errors:
if errors[f]:
num_errors += len(errors[f])
report.append("-" * 80 + "\n")
report.append("File: %s\n" % f)
report.append("-" * 80 + "\n")
report.append("\n".join(errors[f]) + "\n")
report = ("TensorFlow 2.0 Upgrade Script\n"
"-----------------------------\n"
"Converted %d files\n" % files_processed +
"Detected %d issues that require attention" % num_errors + "\n" +
"-" * 80 + "\n") + "".join(report)
detailed_report_header = "=" * 80 + "\n"
detailed_report_header += "Detailed log follows:\n\n"
detailed_report_header += "=" * 80 + "\n"
with open(report_filename, "w") as report_file:
report_file.write(report)
report_file.write(detailed_report_header)
report_file.write(report_text)
if args.print_all:
print(report)
print(detailed_report_header)
print(report_text)
else:
print(report)
print("\nMake sure to read the detailed log %r\n" % report_filename)
if __name__ == "__main__":
main()
tensorflow/tools/compatibility/tf_upgrade_v2_safety.py
+58
View File
@@ -0,0 +1,58 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Upgrader for Python scripts from 1.* to 2.0 TensorFlow using SAFETY mode."""
from tensorflow.tools.compatibility import all_renames_v2
from tensorflow.tools.compatibility import ast_edits
from tensorflow.tools.compatibility import module_deprecations_v2
class TFAPIChangeSpec(ast_edits.APIChangeSpec):
"""List of maps that describe what changed in the API."""
def __init__(self):
self.function_keyword_renames = {}
self.symbol_renames = {}
self.change_to_function = {}
self.function_reorders = {}
self.function_warnings = {}
self.function_transformers = {}
self.module_deprecations = module_deprecations_v2.MODULE_DEPRECATIONS
## Inform about the addons mappings
for symbol, replacement in all_renames_v2.addons_symbol_mappings.items():
warning = (
ast_edits.WARNING, (
"(Manual edit required) `{}` has been migrated to `{}` in "
"TensorFlow Addons. The API spec may have changed during the "
"migration. Please see https://github.com/tensorflow/addons "
"for more info.").format(symbol, replacement))
self.function_warnings[symbol] = warning
# List module renames. If changed, please update max_submodule_depth.
self.import_renames = {
"tensorflow":
ast_edits.ImportRename(
"tensorflow.compat.v1",
excluded_prefixes=[
"tensorflow.contrib", "tensorflow.flags",
"tensorflow.compat",
"tensorflow.compat.v1", "tensorflow.compat.v2",
"tensorflow.google"
],
)
}
# Needs to be updated if self.import_renames is changed.
self.max_submodule_depth = 2
tensorflow/tools/compatibility/tf_upgrade_v2_safety_test.py
+198
View File
@@ -0,0 +1,198 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tf 2.0 upgrader in safety mode."""
import io
from tensorflow.python.framework import test_util
from tensorflow.python.platform import test as test_lib
from tensorflow.tools.compatibility import ast_edits
from tensorflow.tools.compatibility import tf_upgrade_v2_safety
class TfUpgradeV2SafetyTest(test_util.TensorFlowTestCase):
def _upgrade(self, old_file_text):
in_file = io.StringIO(old_file_text)
out_file = io.StringIO()
upgrader = ast_edits.ASTCodeUpgrader(tf_upgrade_v2_safety.TFAPIChangeSpec())
count, report, errors = (
upgrader.process_opened_file("test.py", in_file,
"test_out.py", out_file))
return count, report, errors, out_file.getvalue()
def testContribWarning(self):
text = "tf.contrib.foo()"
_, report, _, _ = self._upgrade(text)
expected_info = "tf.contrib will not be distributed"
self.assertIn(expected_info, report)
def testTensorFlowImport(self):
text = "import tensorflow as tf"
expected_text = ("import tensorflow.compat.v1 as tf")
_, _, _, new_text = self._upgrade(text)
self.assertEqual(expected_text, new_text)
text = "import tensorflow as tf, other_import as y"
expected_text = ("import tensorflow.compat.v1 as tf, other_import as y")
_, _, _, new_text = self._upgrade(text)
self.assertEqual(expected_text, new_text)
text = "import tensorflow"
expected_text = ("import tensorflow.compat.v1 as tensorflow")
_, _, _, new_text = self._upgrade(text)
self.assertEqual(expected_text, new_text)
text = "import tensorflow.foo"
expected_text = "import tensorflow.compat.v1.foo"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(expected_text, new_text)
text = "import tensorflow.foo as bar"
expected_text = "import tensorflow.compat.v1.foo as bar"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(expected_text, new_text)
def testTensorFlowGoogleImport(self):
text = "import tensorflow.google as tf"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(text, new_text)
text = "import tensorflow.google"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(text, new_text)
text = "import tensorflow.google.compat.v1 as tf"
expected_text = "import tensorflow.google.compat.v1 as tf"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(expected_text, new_text)
text = "import tensorflow.google.compat.v2 as tf"
expected_text = "import tensorflow.google.compat.v2 as tf"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(expected_text, new_text)
def testTensorFlowImportInIndent(self):
text = """
try:
import tensorflow as tf # import line
tf.ones([4, 5])
except AttributeError:
pass
"""
expected_text = """
try:
import tensorflow.compat.v1 as tf # import line
tf.ones([4, 5])
except AttributeError:
pass
"""
_, _, _, new_text = self._upgrade(text)
self.assertEqual(expected_text, new_text)
def testTensorFlowFromImport(self):
text = "from tensorflow import foo"
expected_text = "from tensorflow.compat.v1 import foo"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(expected_text, new_text)
text = "from tensorflow.foo import bar"
expected_text = "from tensorflow.compat.v1.foo import bar"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(expected_text, new_text)
text = "from tensorflow import *"
expected_text = "from tensorflow.compat.v1 import *"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(expected_text, new_text)
def testTensorFlowImportAlreadyHasCompat(self):
text = "import tensorflow.compat.v1 as tf"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(text, new_text)
text = "import tensorflow.compat.v2 as tf"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(text, new_text)
text = "from tensorflow.compat import v2 as tf"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(text, new_text)
def testTensorFlowGoogleFromImport(self):
text = "from tensorflow.google.compat import v1 as tf"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(text, new_text)
text = "from tensorflow.google.compat import v2 as tf"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(text, new_text)
def testTensorFlowDontChangeContrib(self):
text = "import tensorflow.contrib as foo"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(text, new_text)
text = "from tensorflow import contrib"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(text, new_text)
def test_contrib_to_addons_move(self):
small_mapping = {
"tf.contrib.layers.poincare_normalize":
"tfa.layers.PoincareNormalize",
"tf.contrib.layers.maxout":
"tfa.layers.Maxout",
"tf.contrib.layers.group_norm":
"tfa.layers.GroupNormalization",
"tf.contrib.layers.instance_norm":
"tfa.layers.InstanceNormalization",
}
for symbol, replacement in small_mapping.items():
text = "{}('stuff', *args, **kwargs)".format(symbol)
_, report, _, _ = self._upgrade(text)
self.assertIn(replacement, report)
if __name__ == "__main__":
test_lib.main()
def testTensorFlowDontChangeContrib(self):
text = "import tensorflow.contrib as foo"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(text, new_text)
text = "from tensorflow import contrib"
_, _, _, new_text = self._upgrade(text)
self.assertEqual(text, new_text)
def test_contrib_to_addons_move(self):
small_mapping = {
"tf.contrib.layers.poincare_normalize":
"tfa.layers.PoincareNormalize",
"tf.contrib.layers.maxout":
"tfa.layers.Maxout",
"tf.contrib.layers.group_norm":
"tfa.layers.GroupNormalization",
"tf.contrib.layers.instance_norm":
"tfa.layers.InstanceNormalization",
}
for symbol, replacement in small_mapping.items():
text = "{}('stuff', *args, **kwargs)".format(symbol)
_, report, _, _ = self._upgrade(text)
self.assertIn(replacement, report)
if __name__ == "__main__":
test_lib.main()
tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+2235
View File
File diff suppressed because it is too large Load Diff
tensorflow/tools/compatibility/update/BUILD
+43
View File
@@ -0,0 +1,43 @@
load("@xla//third_party/rules_python/python:py_binary.bzl", "py_binary")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//visibility:private"],
licenses = ["notice"],
)
py_binary(
name = "generate_v2_renames_map",
srcs = ["generate_v2_renames_map.py"],
strict_deps = True,
deps = [
"//tensorflow:tensorflow_py",
"//tensorflow/python:modules_with_exports",
"//tensorflow/python:no_contrib",
"//tensorflow/python/lib/io:file_io",
"//tensorflow/python/util:tf_decorator_py",
"//tensorflow/python/util:tf_export",
"//tensorflow/tools/common:public_api",
"//tensorflow/tools/common:traverse",
"//tensorflow/tools/compatibility:all_renames_v2",
"@absl_py//absl:app",
],
)
py_binary(
name = "generate_v2_reorders_map",
srcs = ["generate_v2_reorders_map.py"],
strict_deps = True,
deps = [
"//tensorflow:tensorflow_py",
"//tensorflow/python:no_contrib",
"//tensorflow/python/lib/io:file_io",
"//tensorflow/python/util:tf_decorator_py",
"//tensorflow/python/util:tf_export",
"//tensorflow/python/util:tf_inspect",
"//tensorflow/tools/common:public_api",
"//tensorflow/tools/common:traverse",
"//tensorflow/tools/compatibility:tf_upgrade_v2_lib",
"@absl_py//absl:app",
],
)
tensorflow/tools/compatibility/update/generate_v2_renames_map.py
+201
View File
@@ -0,0 +1,201 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# pylint: disable=line-too-long
"""Script for updating tensorflow/tools/compatibility/renames_v2.py.
To update renames_v2.py, run:
bazel run tensorflow/tools/compatibility/update:generate_v2_renames_map
Afterwards, you need to update reoders_v2.py by running:
bazel run tensorflow/tools/compatibility/update:generate_v2_reorders_map
"""
# pylint: enable=line-too-long
import sys
from absl import app
import tensorflow as tf
from tensorflow import python as tf_python # pylint: disable=unused-import
from tensorflow.python import modules_with_exports # pylint: disable=unused-import
from tensorflow.python.lib.io import file_io
from tensorflow.python.util import tf_decorator
from tensorflow.python.util import tf_export
from tensorflow.tools.common import public_api
from tensorflow.tools.common import traverse
from tensorflow.tools.compatibility import all_renames_v2
# This import is needed so that TensorFlow python modules are in sys.modules.
_OUTPUT_FILE_PATH = 'third_party/tensorflow/tools/compatibility/renames_v2.py'
_FILE_HEADER = """# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# pylint: disable=line-too-long
\"\"\"List of renames to apply when converting from TF 1.0 to TF 2.0.
THIS FILE IS AUTOGENERATED: To update, please run:
bazel run tensorflow/tools/compatibility/update:generate_v2_renames_map
This file should be updated whenever endpoints are deprecated.
\"\"\"
"""
def get_canonical_name(v2_names, v1_name):
if v2_names:
return v2_names[0]
return 'compat.v1.%s' % v1_name
def get_all_v2_names():
"""Get a set of function/class names available in TensorFlow 2.0."""
v2_names = set() # All op names in TensorFlow 2.0
def visit(unused_path, unused_parent, children):
"""Visitor that collects TF 2.0 names."""
for child in children:
_, attr = tf_decorator.unwrap(child[1])
api_names_v2 = tf_export.get_v2_names(attr)
for name in api_names_v2:
v2_names.add(name)
visitor = public_api.PublicAPIVisitor(visit)
visitor.do_not_descend_map['tf'].append('contrib')
visitor.private_map['tf.compat'] = ['v1', 'v2']
traverse.traverse(tf.compat.v2, visitor)
return v2_names
def collect_constant_renames():
"""Looks for constants that need to be renamed in TF 2.0.
Returns:
Set of tuples of the form (current name, new name).
"""
renames = set()
for module in sys.modules.copy().values():
try:
constants_v1_list = tf_export.get_v1_constants(module)
constants_v2_list = tf_export.get_v2_constants(module)
except: # pylint: disable=bare-except
pass
# _tf_api_constants attribute contains a list of tuples:
# (api_names_list, constant_name)
# We want to find API names that are in V1 but not in V2 for the same
# constant_names.
# First, we convert constants_v1_list and constants_v2_list to
# dictionaries for easier lookup.
constants_v1 = {constant_name: api_names
for api_names, constant_name in constants_v1_list}
constants_v2 = {constant_name: api_names
for api_names, constant_name in constants_v2_list}
# Second, we look for names that are in V1 but not in V2.
for constant_name, api_names_v1 in constants_v1.items():
api_names_v2 = constants_v2[constant_name]
for name in api_names_v1:
if name not in api_names_v2:
renames.add((name, get_canonical_name(api_names_v2, name)))
return renames
def collect_function_renames():
"""Looks for functions/classes that need to be renamed in TF 2.0.
Returns:
Set of tuples of the form (current name, new name).
"""
# Set of rename lines to write to output file in the form:
# 'tf.deprecated_name': 'tf.canonical_name'
renames = set()
all_v2_names = get_all_v2_names()
def visit(unused_path, unused_parent, children):
"""Visitor that collects rename strings to add to rename_line_set."""
for child in children:
_, attr = tf_decorator.unwrap(child[1])
api_names_v1 = [
name for name in tf_export.get_v1_names(attr)
if '.__internal__.' not in name
]
api_names_v2 = tf_export.get_v2_names(attr)
if not api_names_v2:
# It is possible that a different function is exported with the same
# name. For e.g. when creating a different function to rename arguments.
# Determine if this is the case to not do a useless rename to compat.v1
# for the function and its aliases.
# Note that unsafe v1 to v2 renames created here are overridden by the
# manual_symbol_renames in all_renames_v2.py.
api_names_v2 = [name for name in api_names_v1 if name in all_v2_names]
deprecated_api_names = set(api_names_v1) - set(api_names_v2)
for name in deprecated_api_names:
renames.add((name, get_canonical_name(api_names_v2, name)))
visitor = public_api.PublicAPIVisitor(visit)
visitor.do_not_descend_map['tf'].append('contrib')
visitor.private_map['tf.compat'] = ['v1', 'v2']
traverse.traverse(tf.version, visitor)
traverse.traverse(tf.compat.v1, visitor)
traverse.traverse(tf.compat.v2, visitor)
return renames
def get_rename_line(name, canonical_name):
return ' \'tf.%s\':\n \'tf.%s\'' % (name, canonical_name)
def update_renames_v2(output_file_path):
"""Writes a Python dictionary mapping deprecated to canonical API names.
Args:
output_file_path: File path to write output to. Any existing contents
would be replaced.
"""
function_renames = collect_function_renames()
constant_renames = collect_constant_renames()
all_renames = function_renames.union(constant_renames)
manual_renames = all_renames_v2.manual_symbol_renames
# List of rename lines to write to output file in the form:
# 'tf.deprecated_name': 'tf.canonical_name'
rename_lines = [
get_rename_line(name, canonical_name)
for name, canonical_name in all_renames
if 'tf.' + name not in manual_renames
]
renames_file_text = '%srenames = {\n%s\n}\n' % (
_FILE_HEADER, ',\n'.join(sorted(rename_lines)))
file_io.write_string_to_file(output_file_path, renames_file_text)
def main(unused_argv):
update_renames_v2(_OUTPUT_FILE_PATH)
if __name__ == '__main__':
app.run(main=main)
tensorflow/tools/compatibility/update/generate_v2_reorders_map.py
+202
View File
@@ -0,0 +1,202 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# pylint: disable=line-too-long
"""Script for updating tensorflow/tools/compatibility/reorders_v2.py.
To update reorders_v2.py, run:
bazel run tensorflow/tools/compatibility/update:generate_v2_reorders_map
"""
# pylint: enable=line-too-long
from absl import app
import tensorflow as tf
from tensorflow import python as tf_python # pylint: disable=unused-import
from tensorflow.python.lib.io import file_io
from tensorflow.python.util import tf_decorator
from tensorflow.python.util import tf_export
from tensorflow.python.util import tf_inspect
from tensorflow.tools.common import public_api
from tensorflow.tools.common import traverse
from tensorflow.tools.compatibility import tf_upgrade_v2
# This import is needed so that TensorFlow python modules are in sys.modules.
_OUTPUT_FILE_PATH = 'third_party/tensorflow/tools/compatibility/reorders_v2.py'
_FILE_HEADER = """# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# pylint: disable=line-too-long
\"\"\"List of renames to apply when converting from TF 1.0 to TF 2.0.
THIS FILE IS AUTOGENERATED: To update, please run:
bazel run tensorflow/tools/compatibility/update:generate_v2_reorders_map
This file should be updated whenever a function is added to
self.reordered_function_names in tf_upgrade_v2.py.
\"\"\"
"""
def collect_function_arg_names(function_names, return_all_args_function_names,
function_renames):
"""Determines argument names for reordered function signatures.
Args:
function_names: Functions to collect arguments for.
return_all_args_function_names: Functions to collect all argument names for.
function_renames: Function renames between v1 and v2.
Returns:
Dictionary mapping function names to a list of argument names. Each argument
name list can have leading `None` elements to indicate that some of the
function arguments did not change between v1 and v2.
"""
function_name_v1_to_attr = {}
function_name_v2_to_attr = {}
def visit(unused_path, unused_parent, children):
"""Visitor that collects arguments for reordered functions."""
for child in children:
_, attr = tf_decorator.unwrap(child[1])
api_names_v1 = ['tf.' + name for name in tf_export.get_v1_names(attr)]
if any(name in function_names for name in api_names_v1):
for name in api_names_v1:
function_name_v1_to_attr[name] = attr
api_names_v2 = ['tf.' + name for name in tf_export.get_v2_names(attr)]
for name in api_names_v2:
function_name_v2_to_attr[name] = attr
visitor = public_api.PublicAPIVisitor(visit)
visitor.do_not_descend_map['tf'].append('contrib')
visitor.private_map['tf.compat'] = ['v1', 'v2']
traverse.traverse(tf.compat.v1, visitor)
traverse.traverse(tf.compat.v2, visitor)
def get_arguments_list(attr):
if tf_inspect.isclass(attr):
# Get constructor arguments if attr is a class
arg_list = tf_inspect.getargspec(
getattr(attr, '__init__'))[0]
return arg_list[1:] # skip 'self' argument
else:
# Get function arguments.
# getargspec returns a tuple of (args, varargs, keywords, defaults)
# we just look at args.
return tf_inspect.getargspec(attr)[0]
# Map from reordered function name to its arguments.
function_to_args = {}
if any(name not in function_name_v1_to_attr for name in function_names):
raise ValueError(
'Symbols not found in `tf.compat.v1`: '
f'`{"`, `".join(function_names - function_name_v1_to_attr.keys())}`'
)
for name_v1, attr_v1 in function_name_v1_to_attr.items():
args_v1 = get_arguments_list(attr_v1)
# Per `return_all_args_function_names override`, return all argument names
# without comparing with v2.
if name_v1 in return_all_args_function_names:
function_to_args[name_v1] = args_v1
continue
name_v2 = name_v1
if name_v1 in function_renames:
name_v2 = function_renames[name_v1]
# If the rename is simply mapping `tf.x` to `tf.compat.v1.x`, there is no
# change in the arguments, we shouldn't have it in the list.
if name_v2.startswith('tf.compat.v1.'):
raise ValueError(f'Symbol `{name_v1}` is renamed to `{name_v2}`, '
'no need to add keyword argument names, '
'remove from `reordered_function_names`')
if name_v2 not in function_name_v2_to_attr:
raise ValueError(f'Symbol `{name_v2}` not found in `tf.compat.v2`')
args_v2 = get_arguments_list(function_name_v2_to_attr[name_v2])
# If there is no change in the arguments, we shouldn't have it in the list.
if args_v1 == args_v2:
raise ValueError(f'Symbol `{name_v1}` has no changes in arguments, '
'no need to add keyword argument names, '
'remove from `reordered_function_names`')
# Compare v1/v2 argument names and put `None` as long as they're identical.
needed_arg_names = []
same_so_far = True
for index, arg in enumerate(args_v1):
if same_so_far and index < len(args_v2) and arg == args_v2[index]:
needed_arg_names.append(None)
else:
same_so_far = False
needed_arg_names.append(arg)
function_to_args[name_v1] = needed_arg_names
return function_to_args
def get_reorder_line(name, arg_list):
return ' \'%s\': %s' % (name, str(arg_list))
def update_reorders_v2(output_file_path):
"""Writes a Python dictionary mapping function name to argument order.
Args:
output_file_path: File path to write output to. Any existing contents
would be replaced.
"""
spec = tf_upgrade_v2.TFAPIChangeSpec()
reordered_function_names = spec.reordered_function_names
# We assume that `function_transformers` operate on the keyword arguments, so
# for those we will expand all the arguments
need_kwargs_function_names = spec.function_transformers.keys()
function_renames = spec.symbol_renames
all_reorders = collect_function_arg_names(reordered_function_names,
need_kwargs_function_names,
function_renames)
# List of reorder lines to write to output file in the form:
# 'tf.function_name': ['arg1', 'arg2', ...]
rename_lines = [
get_reorder_line(name, arg_names)
for name, arg_names in all_reorders.items()]
renames_file_text = '%sreorders = {\n%s\n}\n' % (
_FILE_HEADER, ',\n'.join(sorted(rename_lines)))
file_io.write_string_to_file(output_file_path, renames_file_text)
def main(unused_argv):
update_reorders_v2(_OUTPUT_FILE_PATH)
if __name__ == '__main__':
app.run(main=main)