chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,44 @@
|
||||
file(
|
||||
GLOB TEST_INTERP_CASES
|
||||
RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
|
||||
"test_*.py")
|
||||
list(REMOVE_ITEM TEST_INTERP_CASES "test_standalone_custom_event.py")
|
||||
string(REPLACE ".py" "" TEST_INTERP_CASES "${TEST_INTERP_CASES}")
|
||||
|
||||
foreach(target ${TEST_INTERP_CASES})
|
||||
py_test_modules(${target} MODULES ${target})
|
||||
endforeach()
|
||||
|
||||
py_test_modules(
|
||||
test_standalone_executor_no_fast_gc MODULES test_standalone_executor ENVS
|
||||
FLAGS_fast_eager_deletion_mode=false)
|
||||
|
||||
py_test_modules(
|
||||
test_standalone_executor_sequential_run MODULES test_standalone_executor ENVS
|
||||
FLAGS_new_executor_sequential_run=true)
|
||||
|
||||
py_test_modules(
|
||||
test_standalone_executor_serial_run MODULES test_standalone_executor ENVS
|
||||
FLAGS_new_executor_serial_run=true)
|
||||
|
||||
py_test_modules(
|
||||
test_standalone_executor_log_deps MODULES test_standalone_executor ENVS
|
||||
GLOG_v=1 FLAGS_executor_log_deps_every_microseconds=1000)
|
||||
|
||||
py_test_modules(
|
||||
test_standalone_executor_stats MODULES test_standalone_executor ENVS
|
||||
FLAGS_host_trace_level=10 FLAGS_static_executor_perfstat_filepath=./perfstat)
|
||||
|
||||
# These UTs are to temporarily test static build for standalone_executor, will be removed after static build is enabled by default.
|
||||
set(STATIC_BUILD_TESTS
|
||||
test_standalone_controlflow test_standalone_custom_stream
|
||||
test_standalone_custom_event test_standalone_multiply_write
|
||||
test_standalone_executor)
|
||||
|
||||
foreach(STATIC_BUILD_TEST ${STATIC_BUILD_TESTS})
|
||||
py_test_modules(
|
||||
${STATIC_BUILD_TEST}_static_build MODULES ${STATIC_BUILD_TEST} ENVS
|
||||
FLAGS_new_executor_static_build=true)
|
||||
endforeach()
|
||||
|
||||
set_tests_properties(test_standalone_cross_step_overlap PROPERTIES TIMEOUT 30)
|
||||
@@ -0,0 +1,164 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
import paddle
|
||||
from paddle.base import core
|
||||
|
||||
paddle.enable_static()
|
||||
|
||||
|
||||
# test the compatibility of new executor: run old
|
||||
# and new executor twice and check the result.
|
||||
# please override the _get_feeds() and build_program(), run_dygraph_once()
|
||||
class TestCompatibility(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.place = (
|
||||
paddle.CUDAPlace(0)
|
||||
if core.is_compiled_with_cuda()
|
||||
else paddle.CPUPlace()
|
||||
)
|
||||
self.iter_run = 4
|
||||
|
||||
def _get_feed(self):
|
||||
"""return the feeds"""
|
||||
return None
|
||||
|
||||
def build_program(self):
|
||||
def true_func():
|
||||
return paddle.tensor.fill_constant(
|
||||
shape=[1, 2], dtype='float32', value=1
|
||||
), paddle.tensor.fill_constant(shape=[2, 3], dtype='int64', value=1)
|
||||
|
||||
def false_func():
|
||||
return paddle.tensor.fill_constant(
|
||||
shape=[3, 4], dtype='float32', value=3
|
||||
), paddle.tensor.fill_constant(shape=[4, 5], dtype='int64', value=2)
|
||||
|
||||
main_program = paddle.static.Program()
|
||||
startup_program = paddle.static.Program()
|
||||
with paddle.static.program_guard(main_program, startup_program):
|
||||
x = paddle.tensor.fill_constant(
|
||||
shape=[1], dtype='float32', value=0.1
|
||||
)
|
||||
y = paddle.tensor.fill_constant(
|
||||
shape=[1], dtype='float32', value=0.23
|
||||
)
|
||||
pred = paddle.less_than(x, y)
|
||||
out = paddle.static.nn.cond(pred, true_func, false_func)
|
||||
# out is a tuple containing 2 tensors
|
||||
return main_program, startup_program, out
|
||||
|
||||
def _run(self, feed):
|
||||
paddle.seed(2020)
|
||||
|
||||
main_program, startup_program, fetch_vars = self.build_program()
|
||||
|
||||
exe = paddle.static.Executor(self.place)
|
||||
exe.run(startup_program)
|
||||
ret = []
|
||||
for i in range(self.iter_run):
|
||||
ret.append(exe.run(main_program, feed=feed, fetch_list=fetch_vars))
|
||||
return ret
|
||||
|
||||
def run_dygraph_once(self, feed):
|
||||
x = paddle.tensor.fill_constant(shape=[1], dtype='float32', value=0.1)
|
||||
y = paddle.tensor.fill_constant(shape=[1], dtype='float32', value=0.23)
|
||||
if x < y:
|
||||
out = [
|
||||
paddle.tensor.fill_constant(
|
||||
shape=[1, 2], dtype='float32', value=1
|
||||
).numpy(),
|
||||
paddle.tensor.fill_constant(
|
||||
shape=[2, 3], dtype='int64', value=1
|
||||
).numpy(),
|
||||
]
|
||||
else:
|
||||
out = [
|
||||
paddle.tensor.fill_constant(
|
||||
shape=[3, 4], dtype='float32', value=3
|
||||
).numpy(),
|
||||
paddle.tensor.fill_constant(
|
||||
shape=[4, 5], dtype='int64', value=2
|
||||
).numpy(),
|
||||
]
|
||||
return out
|
||||
|
||||
def run_dygraph(self, feed):
|
||||
ret = []
|
||||
for _ in range(self.iter_run):
|
||||
ret.append(self.run_dygraph_once(feed))
|
||||
return ret
|
||||
|
||||
def run_new_executor(self, feed):
|
||||
out = self._run(feed)
|
||||
return out
|
||||
|
||||
def test_with_feed(self):
|
||||
feed = self._get_feed()
|
||||
paddle.enable_static()
|
||||
res = self.run_new_executor(feed)
|
||||
paddle.disable_static()
|
||||
|
||||
gt = self.run_dygraph(feed)
|
||||
|
||||
for x, y in zip(gt, res):
|
||||
if isinstance(x, list):
|
||||
for tx, ty in zip(x, y):
|
||||
np.testing.assert_array_equal(tx, ty)
|
||||
elif isinstance(x, np.ndarray):
|
||||
np.testing.assert_array_equal(x, y)
|
||||
else:
|
||||
raise Exception("Not Implement!")
|
||||
|
||||
|
||||
class TestWhile(TestCompatibility):
|
||||
def _get_feed(self):
|
||||
"""return the feeds"""
|
||||
return None
|
||||
|
||||
def build_program(self):
|
||||
def cond(i, ten):
|
||||
return i < ten
|
||||
|
||||
def body(i, ten):
|
||||
i = i + 1
|
||||
return [i, ten]
|
||||
|
||||
main_program = paddle.static.default_main_program()
|
||||
startup_program = paddle.static.default_startup_program()
|
||||
with paddle.static.program_guard(main_program, startup_program):
|
||||
i = paddle.full(
|
||||
shape=[1], fill_value=0, dtype='int64'
|
||||
) # loop counter
|
||||
ten = paddle.full(
|
||||
shape=[1], fill_value=10, dtype='int64'
|
||||
) # loop length
|
||||
i, ten = paddle.static.nn.while_loop(cond, body, [i, ten])
|
||||
|
||||
exe = paddle.static.Executor(paddle.CPUPlace())
|
||||
return main_program, startup_program, i
|
||||
|
||||
def run_dygraph_once(self, feed):
|
||||
i = 1
|
||||
while i < 10:
|
||||
i = i + 1
|
||||
return [i]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,83 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
import paddle
|
||||
from paddle import static
|
||||
|
||||
paddle.enable_static()
|
||||
|
||||
|
||||
class TestCrossStepOverlap(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.shape = [16, 513, 513, 19]
|
||||
self.x_value = 2
|
||||
self.y_value = 3
|
||||
self.overlap_op_num = 1500
|
||||
self.step_num = 3
|
||||
|
||||
def test_cross_step_overlap(self):
|
||||
if not paddle.base.core.is_compiled_with_cuda():
|
||||
return
|
||||
|
||||
# In this test case, z=x+y is calculated in the default stream,
|
||||
# and at the same time, numerous reduce_min ops that output to y
|
||||
# are executed in another stream (i.e., the custom stream).
|
||||
# These reduce_min ops are carefully designed that their kernel
|
||||
# calculation will overlap with the fill_constant kernels (output
|
||||
# to x and y) in the next step, and therefore cross-step multi-stream
|
||||
# synchronization is required. An Event should be recorded after the
|
||||
# last reduce_min in the first step and waited before the fill_constant
|
||||
# in the second step. Otherwise, the result of z will be wrong.
|
||||
with paddle.pir_utils.OldIrGuard():
|
||||
program = static.Program()
|
||||
with static.program_guard(program):
|
||||
x = paddle.full(
|
||||
self.shape, fill_value=self.x_value, dtype='float64'
|
||||
)
|
||||
y = paddle.full(
|
||||
self.shape, fill_value=self.y_value, dtype='float64'
|
||||
)
|
||||
z = paddle.add(x, y)
|
||||
|
||||
block = program.global_block()
|
||||
block.var(x.name).desc.set_persistable(True)
|
||||
block.var(y.name).desc.set_persistable(True)
|
||||
for i in range(self.overlap_op_num):
|
||||
block.append_op(
|
||||
type='reduce_min',
|
||||
inputs={'X': x.name},
|
||||
outputs={'Out': y.name},
|
||||
attrs={'axis': 0, 'keepdim': True},
|
||||
)
|
||||
block.ops[-1].dist_attr.execution_stream = "custom"
|
||||
|
||||
exe = static.Executor()
|
||||
results = []
|
||||
for i in range(self.step_num):
|
||||
result = exe.run(program, fetch_list=[z])
|
||||
results.append(result)
|
||||
|
||||
for result in results:
|
||||
self.assertAlmostEqual(
|
||||
np.sum(result),
|
||||
(self.x_value + self.y_value) * np.prod(self.shape),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,199 @@
|
||||
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import unittest
|
||||
|
||||
import paddle
|
||||
from paddle.base import core
|
||||
from paddle.base.executor import _add_feed_fetch_ops, _StandaloneExecutor
|
||||
from paddle.distributed.passes.pass_utils import (
|
||||
_add_event_dependency,
|
||||
set_skip_gc_vars,
|
||||
split_program,
|
||||
)
|
||||
|
||||
paddle.enable_static()
|
||||
|
||||
|
||||
def build_program():
|
||||
main_program = paddle.static.Program()
|
||||
startup_program = paddle.static.Program()
|
||||
|
||||
with (
|
||||
paddle.static.program_guard(main_program, startup_program),
|
||||
# data -> [matmul] -> out ->[add] -> add_out
|
||||
paddle.static.device_guard('gpu'),
|
||||
):
|
||||
data = paddle.ones([1024, 2048], dtype='float32', name='data')
|
||||
weight = paddle.randn([2048, 2048], name='weight') # gpu
|
||||
matmul_out = data @ weight
|
||||
bias = paddle.ones([1024, 2048], dtype='float32', name='bias')
|
||||
add_out = paddle.add(matmul_out, bias, name='add_out')
|
||||
# add_out -> [sub] -> sub_out -> [silu] -> silu_out
|
||||
sub_out = paddle.subtract(add_out, data, name='sub_out')
|
||||
silu_out = paddle.nn.functional.silu(sub_out, name='silu_out')
|
||||
bias_1 = paddle.add(bias, sub_out, name='bias_1')
|
||||
out_before = paddle.nn.functional.silu(bias_1, name='out_before')
|
||||
out_last = paddle.subtract(silu_out, data, name='out_last')
|
||||
out_last2 = out_last @ weight
|
||||
|
||||
out = paddle.add(out_before, out_last2, name='out')
|
||||
mean = paddle.mean(out, name='mean_out')
|
||||
|
||||
return main_program, startup_program, [mean]
|
||||
|
||||
|
||||
class TestManualEvent(unittest.TestCase):
|
||||
"""
|
||||
fill_constant(def) gaussian_random(def)
|
||||
| | | |
|
||||
| | matmul_v2(s1) fill_constant(def)
|
||||
| | | | | |
|
||||
| | elementwise_add(s1) |
|
||||
| | | |
|
||||
| elementwise_sub(s1) |
|
||||
| | | |
|
||||
| silu(s1) elementwise_add(s1)
|
||||
| | |
|
||||
elementwise_sub(s1) silu(s1)
|
||||
| |
|
||||
matmul_v2(s1) |
|
||||
| | ---split prog----
|
||||
elementwise_add(s2)
|
||||
|
|
||||
reduce_mean(s2)
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.steps = 3
|
||||
self.place_desc = (
|
||||
paddle.CUDAPlace(0)
|
||||
if core.is_compiled_with_cuda()
|
||||
else paddle.CPUPlace()
|
||||
)
|
||||
self.place = core.Place()
|
||||
self.place.set_place(self.place_desc)
|
||||
|
||||
def set_custom_stream(self, prog):
|
||||
op_index_for_stream1 = [2, 4, 5, 6, 7, 8, 9, 10]
|
||||
op_index_for_stream2 = [11, 12]
|
||||
ops = prog.global_block().ops
|
||||
for op_index in op_index_for_stream1:
|
||||
ops[op_index].dist_attr.execution_stream = "s1"
|
||||
ops[op_index].dist_attr.stream_priority = 0
|
||||
for op_index in op_index_for_stream2:
|
||||
ops[op_index].dist_attr.execution_stream = "s2"
|
||||
ops[op_index].dist_attr.stream_priority = -1
|
||||
|
||||
def split_program(self, prog, apply_manual_event=False):
|
||||
# split two subprograms
|
||||
waiter_recorder_events_map = {11: [8, 10]}
|
||||
prog_block = prog.global_block()
|
||||
ops = prog_block.ops
|
||||
if apply_manual_event:
|
||||
for waiter, recorders in waiter_recorder_events_map.items():
|
||||
for recorder in recorders:
|
||||
_add_event_dependency(ops[recorder], ops[waiter])
|
||||
main_progs, _, _ = split_program(prog, [11])
|
||||
return main_progs
|
||||
|
||||
def create_standalone_exe(self, main_progs, startup_progs, fetch_list):
|
||||
micro_batch_num = 1
|
||||
job_list = []
|
||||
prog_num = len(main_progs)
|
||||
|
||||
if prog_num == 1: # single prog
|
||||
main_progs[0] = _add_feed_fetch_ops(
|
||||
main_progs[0],
|
||||
[],
|
||||
fetch_list,
|
||||
"feed",
|
||||
"fetch",
|
||||
use_fetch_v2=True,
|
||||
)
|
||||
else:
|
||||
main_progs[-1] = _add_feed_fetch_ops(
|
||||
main_progs[-1],
|
||||
[],
|
||||
fetch_list,
|
||||
"feed",
|
||||
"fetch",
|
||||
use_fetch_v2=True,
|
||||
)
|
||||
|
||||
# create jobs
|
||||
for program_id in range(prog_num):
|
||||
job = core.Job(f"prog_{program_id}")
|
||||
job_list.append(job)
|
||||
|
||||
job_types = []
|
||||
for program_id in range(prog_num):
|
||||
job_types.append(f"prog_{program_id}")
|
||||
type_to_program = set_skip_gc_vars(
|
||||
micro_batch_num, job_types, main_progs, job_list
|
||||
)
|
||||
|
||||
for type in type_to_program.keys():
|
||||
type_to_program[type] = type_to_program[type].desc
|
||||
plan = core.Plan(job_list, type_to_program)
|
||||
scope = core.Scope()
|
||||
main_exe = _StandaloneExecutor(self.place, plan, scope)
|
||||
return main_exe
|
||||
|
||||
def run_program(
|
||||
self,
|
||||
apply_custom_stream=False,
|
||||
split_prog=False,
|
||||
apply_manual_event=False,
|
||||
):
|
||||
paddle.seed(2022)
|
||||
main_program, startup_program, fetch_list = build_program()
|
||||
self.assertEqual(len(startup_program.global_block().ops), 0)
|
||||
|
||||
if apply_custom_stream:
|
||||
self.set_custom_stream(main_program)
|
||||
main_progs = [main_program]
|
||||
startup_progs = [startup_program]
|
||||
if apply_custom_stream and split_prog:
|
||||
main_progs = self.split_program(main_program, apply_manual_event)
|
||||
outs = []
|
||||
exe = self.create_standalone_exe(main_progs, startup_progs, fetch_list)
|
||||
for i in range(self.steps):
|
||||
outs.append(exe.run(feed_names=[]))
|
||||
return outs
|
||||
|
||||
def test_result(self):
|
||||
if not core.is_compiled_with_cuda():
|
||||
return
|
||||
with paddle.pir_utils.OldIrGuard():
|
||||
baselines = self.run_program()
|
||||
stream_outs = self.run_program(apply_custom_stream=True)
|
||||
split_outs = self.run_program(
|
||||
apply_custom_stream=True, split_prog=True
|
||||
)
|
||||
manual_outs = self.run_program(
|
||||
apply_custom_stream=True,
|
||||
split_prog=True,
|
||||
apply_manual_event=True,
|
||||
)
|
||||
for bl, out0, out1, out2 in zip(
|
||||
baselines, stream_outs, split_outs, manual_outs
|
||||
):
|
||||
self.assertEqual(bl[0], out0[0])
|
||||
self.assertEqual(bl[0], out2[0])
|
||||
# self.assertNotEqual(bl[0], out1[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,98 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.append("../legacy_test")
|
||||
from test_standalone_executor import build_program
|
||||
from utils import compare_legacy_with_pt
|
||||
|
||||
import paddle
|
||||
from paddle.base import core
|
||||
|
||||
paddle.enable_static()
|
||||
|
||||
|
||||
class TestCustomStream(unittest.TestCase):
|
||||
"""
|
||||
fill_constant(cpu) gaussian_random
|
||||
| | | |
|
||||
| | matmul_v2(s1) fill_constant
|
||||
| | | | |
|
||||
| | elementwise_add(s1) |
|
||||
| | | |
|
||||
| elementwise_sub(cpu) |
|
||||
| | | |
|
||||
| silu(cpu) elementwise_add(s2)
|
||||
| | |
|
||||
elementwise_sub(s1) silu(s2)
|
||||
| |
|
||||
elementwise_add(s2)
|
||||
|
|
||||
reduce_mean(s2)
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.steps = 3
|
||||
|
||||
def set_custom_stream(self, prog):
|
||||
op_index_for_stream1 = [2, 4, 9]
|
||||
op_index_for_stream2 = [7, 8, 10, 11]
|
||||
ops = prog.global_block().ops
|
||||
for op_index in op_index_for_stream1:
|
||||
if paddle.framework.in_pir_mode():
|
||||
ops[op_index].set_execution_stream("s1")
|
||||
ops[op_index].set_scheduling_priority(0)
|
||||
else:
|
||||
ops[op_index].dist_attr.execution_stream = "s1"
|
||||
ops[op_index].dist_attr.stream_priority = 0
|
||||
for op_index in op_index_for_stream2:
|
||||
if paddle.framework.in_pir_mode():
|
||||
ops[op_index].set_execution_stream("s2")
|
||||
ops[op_index].set_scheduling_priority(-1)
|
||||
else:
|
||||
ops[op_index].dist_attr.execution_stream = "s2"
|
||||
ops[op_index].dist_attr.stream_priority = -1
|
||||
|
||||
def run_program(self, apply_custom_stream=False):
|
||||
paddle.seed(2022)
|
||||
main_program, startup_program, fetch_list = build_program()
|
||||
self.assertEqual(len(startup_program.global_block().ops), 0)
|
||||
|
||||
if apply_custom_stream:
|
||||
self.set_custom_stream(main_program)
|
||||
with paddle.static.program_guard(main_program, startup_program):
|
||||
exe = paddle.static.Executor(paddle.CUDAPlace(0))
|
||||
scope = core.Scope()
|
||||
outs = []
|
||||
for i in range(self.steps):
|
||||
outs.append(
|
||||
exe.run(main_program, scope=scope, fetch_list=fetch_list)
|
||||
)
|
||||
return outs
|
||||
|
||||
@compare_legacy_with_pt
|
||||
def test_result(self):
|
||||
if not core.is_compiled_with_cuda():
|
||||
return
|
||||
|
||||
baselines = self.run_program()
|
||||
outs = self.run_program(apply_custom_stream=True)
|
||||
for bl, out in zip(baselines, outs):
|
||||
self.assertEqual(bl[0], out[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,374 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
os.environ['FLAGS_use_stream_safe_cuda_allocator'] = "true"
|
||||
import json
|
||||
import shutil
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
from utils import static_guard
|
||||
|
||||
import paddle
|
||||
from paddle.base import core
|
||||
from paddle.profiler import profiler
|
||||
|
||||
paddle.enable_static()
|
||||
|
||||
|
||||
def build_program():
|
||||
main_program = paddle.static.Program()
|
||||
startup_program = paddle.static.Program()
|
||||
|
||||
with paddle.static.program_guard(main_program, startup_program):
|
||||
with paddle.static.device_guard('cpu'):
|
||||
data = paddle.ones([4, 64], dtype='float32', name='data')
|
||||
|
||||
# data -> [memcpy_h2d] -> data' -> [matmul] -> out ->[add] -> add_out
|
||||
with paddle.static.device_guard('gpu'):
|
||||
weight = paddle.randn([64, 64], name='weight') # gpu
|
||||
matmul_out = data @ weight # gpus
|
||||
bias = paddle.ones([4, 64], dtype='float32', name='bias')
|
||||
add_out = paddle.add(matmul_out, bias, name='add_out')
|
||||
|
||||
# add_out -> [memcpy_d2h] -> add_out' -> [sub] -> sub_out -> [silu] -> silu_out
|
||||
with paddle.static.device_guard('cpu'):
|
||||
sub_out = paddle.subtract(add_out, data, name='sub_out')
|
||||
silu_out = paddle.nn.functional.silu(sub_out, name='silu_out')
|
||||
|
||||
with paddle.static.device_guard('gpu'):
|
||||
bias_1 = paddle.add(bias, sub_out, name='bias_1')
|
||||
out_before = paddle.nn.functional.silu(bias_1, name='out_before')
|
||||
out_last = paddle.subtract(silu_out, data, name='out_last')
|
||||
|
||||
out = paddle.add(out_before, out_last, name='out')
|
||||
mean = paddle.mean(out, name='mean_out')
|
||||
|
||||
return main_program, startup_program, [mean]
|
||||
|
||||
|
||||
class ExecutorStatisticsTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.iter_n = 3
|
||||
self.place = (
|
||||
paddle.CUDAPlace(0)
|
||||
if core.is_compiled_with_cuda()
|
||||
else paddle.CPUPlace()
|
||||
)
|
||||
self.perf_path = './perfstat'
|
||||
|
||||
def test_executor_statistics(self):
|
||||
self.run_with_statistics(executor='Executor')
|
||||
|
||||
def test_standalone_executor_statistics(self):
|
||||
self.run_with_statistics(executor='StandaloneExecutor')
|
||||
|
||||
def run_with_statistics(self, executor=None):
|
||||
# random failed, skip this testcase
|
||||
return
|
||||
if os.getenv("FLAGS_static_executor_perfstat_filepath") is None:
|
||||
return
|
||||
paddle.seed(2020)
|
||||
# note: startup program is empty
|
||||
main_program, startup_program, fetch_list = build_program()
|
||||
|
||||
scope = paddle.static.Scope()
|
||||
with paddle.static.scope_guard(scope):
|
||||
exe = paddle.static.Executor(self.place)
|
||||
helper_profiler = profiler.Profiler(
|
||||
targets=[profiler.ProfilerTarget.CPU], scheduler=(1, 2)
|
||||
)
|
||||
helper_profiler.start()
|
||||
for i in range(self.iter_n):
|
||||
exe.run(main_program, fetch_list=fetch_list)
|
||||
helper_profiler.step()
|
||||
helper_profiler.stop()
|
||||
|
||||
self.assertTrue(os.path.exists(self.perf_path))
|
||||
with open(self.perf_path, 'r') as load_f:
|
||||
stat_res = json.load(load_f)
|
||||
self.assertTrue(len(stat_res) > 0)
|
||||
|
||||
os.remove(self.perf_path)
|
||||
shutil.rmtree('./profiler_log')
|
||||
|
||||
|
||||
class MultiStreamModelTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.iter_n = 2
|
||||
self.place = (
|
||||
paddle.CUDAPlace(0)
|
||||
if core.is_compiled_with_cuda()
|
||||
else paddle.CPUPlace()
|
||||
)
|
||||
|
||||
def test_result(self):
|
||||
ground_truths = self.run_test(False)
|
||||
res = self.run_test(True)
|
||||
|
||||
for gt, out in zip(ground_truths, res):
|
||||
self.assertEqual(gt[0], out[0])
|
||||
|
||||
def run_test(self, use_new_executor=True):
|
||||
paddle.seed(2020)
|
||||
main_program, startup_program, fetch_list = build_program()
|
||||
|
||||
scope = core.Scope()
|
||||
exe = paddle.static.Executor(self.place)
|
||||
outs = []
|
||||
for i in range(self.iter_n):
|
||||
outs.append(
|
||||
exe.run(main_program, scope=scope, fetch_list=fetch_list)
|
||||
)
|
||||
print(outs)
|
||||
return outs
|
||||
|
||||
|
||||
class SwitchExecutorInterfaceWithFeed(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.place = (
|
||||
paddle.CUDAPlace(0)
|
||||
if core.is_compiled_with_cuda()
|
||||
else paddle.CPUPlace()
|
||||
)
|
||||
self.iter_run = 2
|
||||
|
||||
def build_program(self, is_double=False):
|
||||
main_program = paddle.static.Program()
|
||||
startup_program = paddle.static.Program()
|
||||
with paddle.static.program_guard(main_program, startup_program):
|
||||
a = paddle.static.data(name="a", shape=[2, 2], dtype='float32')
|
||||
b = paddle.ones([2, 2]) * 2
|
||||
t = paddle.static.nn.fc(a, 2)
|
||||
c = t + b
|
||||
if is_double:
|
||||
c = c + c
|
||||
|
||||
return main_program, startup_program, [c]
|
||||
|
||||
def _run(
|
||||
self,
|
||||
feed,
|
||||
use_str=False,
|
||||
is_double=False,
|
||||
add_wrong_fetch=False,
|
||||
use_compiled=False,
|
||||
):
|
||||
paddle.seed(2020)
|
||||
|
||||
main_program, startup_program, fetch_vars = self.build_program(
|
||||
is_double
|
||||
)
|
||||
|
||||
exe = paddle.static.Executor(self.place)
|
||||
exe.run(startup_program)
|
||||
|
||||
if use_compiled:
|
||||
main_program = paddle.static.CompiledProgram(main_program)
|
||||
|
||||
if (
|
||||
use_str and not paddle.framework.in_pir_mode()
|
||||
): # test for fetch name
|
||||
fetch_vars = [x.name for x in fetch_vars]
|
||||
if add_wrong_fetch: # test for wrong fetch type
|
||||
fetch_vars.append(1123)
|
||||
outs = []
|
||||
for i in range(self.iter_run):
|
||||
out = exe.run(main_program, feed=feed, fetch_list=fetch_vars)[0]
|
||||
|
||||
outs.append(out)
|
||||
|
||||
return outs
|
||||
|
||||
def run_dygraph(self, feed):
|
||||
def run_once(is_double):
|
||||
paddle.seed(2020)
|
||||
a = feed['a']
|
||||
a = paddle.to_tensor(a, dtype='float32')
|
||||
b = paddle.ones([2, 2]) * 2
|
||||
t = paddle.nn.Linear(2, 2)(a)
|
||||
c = t + b
|
||||
if is_double:
|
||||
c = c + c
|
||||
return c.numpy()
|
||||
|
||||
out1 = []
|
||||
for i in range(self.iter_run):
|
||||
out1.append(run_once(False))
|
||||
out2 = []
|
||||
for i in range(self.iter_run):
|
||||
out2.append(run_once(True))
|
||||
return [out1, out2]
|
||||
|
||||
def run_new_executor(self, feed, use_compiled=False):
|
||||
# run construct program 1
|
||||
out1 = self._run(
|
||||
feed, use_str=False, is_double=False, use_compiled=use_compiled
|
||||
)
|
||||
# run construct program 2 with same executor
|
||||
out2 = self._run(
|
||||
feed, use_str=True, is_double=True, use_compiled=use_compiled
|
||||
)
|
||||
|
||||
return [out1, out2]
|
||||
|
||||
def test_with_feed(self):
|
||||
data = np.ones([2, 2], dtype="float32")
|
||||
feed = {"a": data, 'fake_input': data}
|
||||
|
||||
with static_guard():
|
||||
res = self.run_new_executor(feed)
|
||||
with paddle.base.dygraph.guard():
|
||||
gt = self.run_dygraph(feed)
|
||||
for x, y in zip(gt, res):
|
||||
np.testing.assert_array_equal(x, y)
|
||||
|
||||
def test_with_error(self):
|
||||
feed = [{'a': np.ones([2, 2], dtype="float32")}]
|
||||
|
||||
with self.assertRaises(TypeError):
|
||||
self._run(feed[0], add_wrong_fetch=True)
|
||||
|
||||
def test_empty_program(self):
|
||||
program = paddle.static.Program()
|
||||
exe = paddle.static.Executor(self.place)
|
||||
for i in range(10):
|
||||
out = exe.run() # old executor
|
||||
|
||||
for i in range(10):
|
||||
print(i, flush=1)
|
||||
out = exe.run(program, feed=None)
|
||||
|
||||
|
||||
class TestException(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.place = paddle.CPUPlace()
|
||||
self.fetch_vars = None
|
||||
|
||||
def build_program(self):
|
||||
main_program = paddle.static.Program()
|
||||
startup_program = paddle.static.Program()
|
||||
with paddle.static.program_guard(main_program, startup_program):
|
||||
w = paddle.rand([10, 3])
|
||||
ids = paddle.static.data(name="id", shape=[5], dtype='int64')
|
||||
data = paddle.static.data(name="data", shape=[3], dtype='float32')
|
||||
emb = paddle.nn.functional.embedding(
|
||||
x=ids, weight=w, sparse=False, name="embedding"
|
||||
)
|
||||
emb = emb + data
|
||||
|
||||
return main_program, startup_program, emb
|
||||
|
||||
def _run(self, feeds):
|
||||
paddle.seed(2020)
|
||||
|
||||
main_program, startup_program, fetch_vars = self.build_program()
|
||||
|
||||
exe = paddle.static.Executor(self.place)
|
||||
exe.run(startup_program)
|
||||
|
||||
for feed in feeds:
|
||||
out = exe.run(main_program, feed=feed, fetch_list=fetch_vars)
|
||||
self.fetch_vars = fetch_vars
|
||||
return out
|
||||
|
||||
def run_new_executor(self, feed):
|
||||
out = self._run(feed)
|
||||
return out
|
||||
|
||||
def test_exception(self):
|
||||
feed = [
|
||||
{
|
||||
'id': np.array([1, 2, 3, 4, 5]).astype(np.int64),
|
||||
'data': np.array([1, 2, 3]).astype(np.float32),
|
||||
},
|
||||
{
|
||||
'id': np.array([1, 2, 3, 4, 11]).astype(np.int64),
|
||||
'data': np.array([1, 2, 3]).astype(np.float32),
|
||||
},
|
||||
]
|
||||
self.assertRaises(ValueError, self.run_new_executor, feed)
|
||||
|
||||
def test_nan(self):
|
||||
flags = {'FLAGS_check_nan_inf': True, 'FLAGS_benchmark': True}
|
||||
paddle.base.set_flags(flags)
|
||||
feed = [
|
||||
{
|
||||
'id': np.array([1, 2, 3, 4, 5]).astype(np.int64),
|
||||
'data': np.array([1, 2, 3]).astype(np.float32),
|
||||
},
|
||||
{
|
||||
'id': np.array([1, 2, 3, 4, 5]).astype(np.int64),
|
||||
'data': np.array([1, 2, 3]).astype(np.float32),
|
||||
},
|
||||
]
|
||||
feed[1]['data'][0] = np.nan
|
||||
self.assertRaises(RuntimeError, self.run_new_executor, feed)
|
||||
|
||||
def test_scope_find_temp_var(self):
|
||||
feed = [
|
||||
{
|
||||
'id': np.array([1, 2, 3, 4, 5]).astype(np.int64),
|
||||
'data': np.array([1, 2, 3]).astype(np.float32),
|
||||
},
|
||||
{
|
||||
'id': np.array([1, 2, 3, 4, 5]).astype(np.int64),
|
||||
'data': np.array([2, 2, 2]).astype(np.float32),
|
||||
},
|
||||
]
|
||||
self.run_new_executor(feed)
|
||||
if not paddle.framework.in_pir_mode():
|
||||
self.assertIsNone(
|
||||
paddle.static.global_scope().find_var(self.fetch_vars.name)
|
||||
)
|
||||
|
||||
|
||||
class TestFetchEmptyTensor(unittest.TestCase):
|
||||
def test_fetch(self):
|
||||
places = []
|
||||
if (
|
||||
os.environ.get('FLAGS_CI_both_cpu_and_gpu', 'False').lower()
|
||||
in ['1', 'true', 'on']
|
||||
or not paddle.base.core.is_compiled_with_cuda()
|
||||
):
|
||||
places.append(paddle.CPUPlace())
|
||||
if paddle.base.core.is_compiled_with_cuda():
|
||||
places.append(paddle.CUDAPlace(0))
|
||||
for place in places:
|
||||
with paddle.static.program_guard(paddle.static.Program()):
|
||||
out = paddle.empty([3, 0])
|
||||
exe = paddle.static.Executor(place)
|
||||
res = exe.run(fetch_list=[out])
|
||||
self.assertEqual(res[0].shape, (3, 0))
|
||||
|
||||
|
||||
class TestInplaceApiWithDataTransform(unittest.TestCase):
|
||||
def test_increment(self):
|
||||
if paddle.base.core.is_compiled_with_cuda():
|
||||
with paddle.base.device_guard("gpu:0"):
|
||||
x = paddle.tensor.fill_constant([1], "float32", 0)
|
||||
with paddle.base.device_guard("cpu"):
|
||||
x = paddle.increment(x)
|
||||
exe = paddle.static.Executor(paddle.CUDAPlace(0))
|
||||
for i in range(10):
|
||||
(a,) = exe.run(
|
||||
paddle.static.default_main_program(), fetch_list=[x]
|
||||
)
|
||||
self.assertEqual(a[0], 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,53 @@
|
||||
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import unittest
|
||||
|
||||
from paddle import static
|
||||
from paddle.distributed.passes import PassContext, new_pass
|
||||
|
||||
|
||||
class TestStandaloneExecutorFThenBPlan(unittest.TestCase):
|
||||
def test_standalone_executor_fthenb_plan(self):
|
||||
config = {}
|
||||
config["num_micro_batches"] = 4
|
||||
pass_context = PassContext()
|
||||
|
||||
startup_program = static.Program()
|
||||
main_program = static.Program()
|
||||
|
||||
pipeline_fthenb_pass = new_pass("pipeline_scheduler_FThenB", config)
|
||||
pipeline_fthenb_pass.apply(
|
||||
[main_program], [startup_program], pass_context
|
||||
)
|
||||
plan = pass_context.get_attr("plan")
|
||||
job_type_list = []
|
||||
for job in plan.job_list():
|
||||
job_type_list.append(job.type())
|
||||
expect_job_type_list = [
|
||||
"forward",
|
||||
"forward",
|
||||
"forward",
|
||||
"forward",
|
||||
"backward",
|
||||
"backward",
|
||||
"backward",
|
||||
"backward",
|
||||
"optimizer",
|
||||
]
|
||||
self.assertEqual(job_type_list, expect_job_type_list)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,57 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import unittest
|
||||
|
||||
from test_standalone_controlflow import TestCompatibility
|
||||
|
||||
import paddle
|
||||
|
||||
# from paddle.base.framework import Program
|
||||
|
||||
paddle.enable_static()
|
||||
|
||||
|
||||
class TestMultiplyWrite(TestCompatibility):
|
||||
def _get_feed(self):
|
||||
"""return the feeds"""
|
||||
return None
|
||||
|
||||
def build_program(self):
|
||||
main_program = paddle.static.Program()
|
||||
startup_program = paddle.static.Program()
|
||||
with paddle.static.program_guard(main_program, startup_program):
|
||||
out = paddle.full((1,), 1)
|
||||
inp1 = paddle.full((1,), 2)
|
||||
inp2 = paddle.full((1,), 3)
|
||||
|
||||
paddle.assign(inp1, out)
|
||||
paddle.assign(inp2, out)
|
||||
return main_program, startup_program, out
|
||||
|
||||
def run_dygraph_once(self, feed):
|
||||
out = paddle.full((1,), 1)
|
||||
inp1 = paddle.full((1,), 2)
|
||||
inp2 = paddle.full((1,), 3)
|
||||
paddle.assign(inp1, out)
|
||||
paddle.assign(inp2, out)
|
||||
return [out.numpy()]
|
||||
|
||||
def setUp(self):
|
||||
self.place = paddle.CPUPlace()
|
||||
self.iter_run = 5
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user