wehub-resource-sync
205 lines
7.9 KiB
Python
205 lines
7.9 KiB
Python
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
# ==============================================================================
|
|
"""Tests for a little bit of strategy_combinations."""
|
|
|
|
from absl.testing import parameterized
|
|
|
|
from tensorflow.python import tf2
|
|
from tensorflow.python.distribute import central_storage_strategy
|
|
from tensorflow.python.distribute import collective_all_reduce_strategy
|
|
from tensorflow.python.distribute import combinations
|
|
from tensorflow.python.distribute import mirrored_strategy
|
|
from tensorflow.python.distribute import one_device_strategy
|
|
from tensorflow.python.distribute import parameter_server_strategy_v2
|
|
from tensorflow.python.distribute import reduce_util
|
|
from tensorflow.python.distribute import strategy_combinations
|
|
from tensorflow.python.distribute import test_util
|
|
from tensorflow.python.distribute import tpu_strategy
|
|
from tensorflow.python.eager import def_function
|
|
from tensorflow.python.framework import constant_op
|
|
from tensorflow.python.ops import array_ops
|
|
from tensorflow.python.platform import test
|
|
|
|
|
|
class StrategyCombinationsTest(test.TestCase, parameterized.TestCase):
|
|
|
|
@combinations.generate(
|
|
combinations.combine(
|
|
strategy=strategy_combinations.two_replica_strategies,
|
|
mode=["graph", "eager"]))
|
|
def testTwoReplicaStrategy(self, strategy):
|
|
with strategy.scope():
|
|
|
|
@def_function.function
|
|
def one():
|
|
return array_ops.identity(1.)
|
|
|
|
one_per_replica = strategy.run(one)
|
|
num_replicas = strategy.reduce(
|
|
reduce_util.ReduceOp.SUM, one_per_replica, axis=None)
|
|
self.assertEqual(self.evaluate(num_replicas), 2.)
|
|
|
|
@combinations.generate(
|
|
combinations.combine(
|
|
strategy=strategy_combinations.four_replica_strategies,
|
|
mode=["graph", "eager"]))
|
|
def testFourReplicaStrategy(self, strategy):
|
|
with strategy.scope():
|
|
|
|
@def_function.function
|
|
def one():
|
|
return array_ops.identity(1.)
|
|
|
|
one_per_replica = strategy.run(one)
|
|
num_replicas = strategy.reduce(
|
|
reduce_util.ReduceOp.SUM, one_per_replica, axis=None)
|
|
self.assertEqual(self.evaluate(num_replicas), 4.)
|
|
|
|
@combinations.generate(
|
|
combinations.combine(
|
|
distribution=[
|
|
strategy_combinations.mirrored_strategy_with_cpu_1_and_2
|
|
],
|
|
mode=["graph", "eager"]))
|
|
def testMirrored2CPUs(self, distribution):
|
|
with distribution.scope():
|
|
one_per_replica = distribution.run(lambda: constant_op.constant(1))
|
|
num_replicas = distribution.reduce(
|
|
reduce_util.ReduceOp.SUM, one_per_replica, axis=None)
|
|
self.assertEqual(2, self.evaluate(num_replicas))
|
|
|
|
|
|
class V1StrategyTest(test.TestCase, parameterized.TestCase):
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
tf2.disable()
|
|
|
|
@combinations.generate(
|
|
combinations.combine(strategy=[
|
|
strategy_combinations.one_device_strategy,
|
|
strategy_combinations.one_device_strategy_gpu,
|
|
strategy_combinations.one_device_strategy_gpu_on_worker_1,
|
|
strategy_combinations.one_device_strategy_on_worker_1
|
|
]))
|
|
def testOneDevice(self, strategy):
|
|
self.assertIsInstance(strategy, one_device_strategy.OneDeviceStrategyV1)
|
|
|
|
@combinations.generate(
|
|
combinations.combine(strategy=[
|
|
strategy_combinations.mirrored_strategy_with_cpu_1_and_2,
|
|
strategy_combinations.mirrored_strategy_with_gpu_and_cpu,
|
|
strategy_combinations.mirrored_strategy_with_one_cpu,
|
|
strategy_combinations.mirrored_strategy_with_one_gpu,
|
|
strategy_combinations.mirrored_strategy_with_two_gpus,
|
|
]))
|
|
def testMirrored(self, strategy):
|
|
self.assertIsInstance(strategy, mirrored_strategy.MirroredStrategyV1)
|
|
|
|
@combinations.generate(
|
|
combinations.combine(strategy=[
|
|
strategy_combinations.multi_worker_mirrored_2x1_cpu,
|
|
strategy_combinations.multi_worker_mirrored_2x1_gpu,
|
|
strategy_combinations.multi_worker_mirrored_2x2_gpu,
|
|
strategy_combinations.multi_worker_mirrored_4x1_cpu,
|
|
]))
|
|
def testMultiWorkerMirrored(self, strategy):
|
|
# MultiWorkerMirroredStrategy combinations only supports V2.
|
|
self.assertIsInstance(
|
|
strategy, collective_all_reduce_strategy.CollectiveAllReduceStrategy)
|
|
|
|
@combinations.generate(
|
|
combinations.combine(strategy=[
|
|
strategy_combinations.central_storage_strategy_with_gpu_and_cpu,
|
|
strategy_combinations.central_storage_strategy_with_two_gpus,
|
|
]))
|
|
def testCentralStorage(self, strategy):
|
|
self.assertIsInstance(strategy,
|
|
central_storage_strategy.CentralStorageStrategyV1)
|
|
|
|
@combinations.generate(
|
|
combinations.combine(strategy=strategy_combinations.tpu_strategies))
|
|
def testTPU(self, strategy):
|
|
self.assertIsInstance(strategy, tpu_strategy.TPUStrategyV1)
|
|
|
|
|
|
class V2StrategyTest(test.TestCase, parameterized.TestCase):
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
tf2.enable()
|
|
|
|
@combinations.generate(
|
|
combinations.combine(strategy=[
|
|
strategy_combinations.one_device_strategy,
|
|
strategy_combinations.one_device_strategy_gpu,
|
|
strategy_combinations.one_device_strategy_gpu_on_worker_1,
|
|
strategy_combinations.one_device_strategy_on_worker_1
|
|
]))
|
|
def testOneDevice(self, strategy):
|
|
self.assertIsInstance(strategy, one_device_strategy.OneDeviceStrategy)
|
|
|
|
@combinations.generate(
|
|
combinations.combine(strategy=[
|
|
strategy_combinations.mirrored_strategy_with_cpu_1_and_2,
|
|
strategy_combinations.mirrored_strategy_with_gpu_and_cpu,
|
|
strategy_combinations.mirrored_strategy_with_one_cpu,
|
|
strategy_combinations.mirrored_strategy_with_one_gpu,
|
|
strategy_combinations.mirrored_strategy_with_two_gpus,
|
|
]))
|
|
def testMirrored(self, strategy):
|
|
self.assertIsInstance(strategy, mirrored_strategy.MirroredStrategy)
|
|
|
|
@combinations.generate(
|
|
combinations.combine(strategy=[
|
|
strategy_combinations.multi_worker_mirrored_2x1_cpu,
|
|
strategy_combinations.multi_worker_mirrored_2x1_gpu,
|
|
strategy_combinations.multi_worker_mirrored_2x2_gpu,
|
|
strategy_combinations.multi_worker_mirrored_4x1_cpu,
|
|
]))
|
|
def testMultiWorkerMirrored(self, strategy):
|
|
self.assertIsInstance(
|
|
strategy, collective_all_reduce_strategy.CollectiveAllReduceStrategy)
|
|
|
|
@combinations.generate(
|
|
combinations.combine(strategy=[
|
|
strategy_combinations.central_storage_strategy_with_gpu_and_cpu,
|
|
strategy_combinations.central_storage_strategy_with_two_gpus,
|
|
]))
|
|
def testCentralStorage(self, strategy):
|
|
self.assertIsInstance(strategy,
|
|
central_storage_strategy.CentralStorageStrategy)
|
|
|
|
@combinations.generate(
|
|
combinations.combine(strategy=strategy_combinations.tpu_strategies))
|
|
def testTPU(self, strategy):
|
|
self.assertIsInstance(
|
|
strategy, (tpu_strategy.TPUStrategy, tpu_strategy.TPUStrategyV2))
|
|
|
|
@combinations.generate(
|
|
combinations.combine(strategy=[
|
|
strategy_combinations.parameter_server_strategy_3worker_2ps_cpu,
|
|
strategy_combinations.parameter_server_strategy_1worker_2ps_cpu,
|
|
strategy_combinations.parameter_server_strategy_3worker_2ps_1gpu,
|
|
strategy_combinations.parameter_server_strategy_1worker_2ps_1gpu,
|
|
]))
|
|
def testParameterServer(self, strategy):
|
|
self.assertIsInstance(
|
|
strategy, parameter_server_strategy_v2.ParameterServerStrategyV2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
test_util.main()
|