chore: import upstream snapshot with attribution

test/sequence/CMakeLists.txt

@@ -0,0 +1,21 @@
+file(
+  GLOB TEST_OPS
+  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
+  "test_*.py")
+string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
+
+foreach(TEST_OP ${TEST_OPS})
+  py_test_modules(${TEST_OP} MODULES ${TEST_OP})
+endforeach()
+
+set(PIR_COVERAGE_TESTS test_sequence_mask)
+
+foreach(PIR_COVERAGE_TEST ${PIR_COVERAGE_TESTS})
+  py_test_modules(${PIR_COVERAGE_TEST}_pir MODULES ${PIR_COVERAGE_TEST} ENVS
+                  FLAGS_enable_pir_in_executor=true)
+  set_tests_properties(${PIR_COVERAGE_TEST}_pir PROPERTIES TIMEOUT 120)
+  message(STATUS "PIR Copied OpTest: ${PIR_COVERAGE_TEST}_pir in sequence test")
+endforeach()
+
+set_tests_properties(test_sequence_conv PROPERTIES TIMEOUT 120)
+set_tests_properties(test_sequence_pool PROPERTIES TIMEOUT 120)

test/sequence/__init__.py

@@ -0,0 +1,13 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

test/sequence/test_sequence_conv.py

@@ -0,0 +1,289 @@
+#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+import unittest
+
+import numpy as np
+from op_test import OpTest
+
+
+def seqconv(
+    x,
+    lod,
+    filter,
+    context_length,
+    context_start,
+    padding_trainable=False,
+    padding_data=None,
+):
+    [T, M] = x.shape
+    col = np.zeros((T, context_length * M)).astype('float32')
+    offset = [0]
+    for seq_len in lod[0]:
+        offset.append(offset[-1] + seq_len)
+    begin_pad = np.max([0, -context_start])
+    for i in range(len(offset) - 1):
+        for j in range(context_length):
+            in_begin = offset[i] + context_start + j
+            in_end = offset[i + 1] + context_start + j
+            out_begin = offset[i]
+            out_end = offset[i + 1]
+            if in_begin < offset[i]:
+                pad_size = np.min(
+                    [offset[i] - in_begin, offset[i + 1] - offset[i]]
+                )
+                if padding_trainable:
+                    sub_w = padding_data[j : j + pad_size, :]
+                    col[
+                        offset[i] : offset[i] + pad_size, j * M : (j + 1) * M
+                    ] = sub_w
+                out_begin = offset[i] + pad_size
+                in_begin = offset[i]
+
+            if in_end > offset[i + 1]:
+                pad_size = np.min(
+                    [in_end - offset[i + 1], offset[i + 1] - offset[i]]
+                )
+                if padding_trainable:
+                    sub_w = padding_data[
+                        begin_pad + context_start + j - pad_size : begin_pad
+                        + context_start
+                        + j,
+                        :,
+                    ]
+                    col[
+                        offset[i + 1] - pad_size : offset[i + 1],
+                        j * M : (j + 1) * M,
+                    ] = sub_w
+                in_end = offset[i + 1]
+                out_end = offset[i + 1] - pad_size
+            if in_end <= in_begin:
+                continue
+            in_sub = x[in_begin:in_end, :]
+            col[out_begin:out_end, j * M : (j + 1) * M] += in_sub
+    return np.dot(col, filter)
+
+
+class TestSeqProject(OpTest):
+    def setUp(self):
+        self.init_test_case()
+        self.op_type = 'sequence_conv'
+
+        if (
+            self.context_length == 1
+            and self.context_start == 0
+            and self.padding_trainable
+        ):
+            print(
+                "If context_start is 0 "
+                "and context_length is 1,"
+                " padding_trainable should be false."
+            )
+            return
+
+        # one level, batch size
+        x = np.random.uniform(
+            0.1, 1, [self.input_size[0], self.input_size[1]]
+        ).astype('float32')
+        w = np.random.uniform(
+            0.1,
+            1,
+            [
+                self.context_length * self.input_size[1],
+                self.output_representation,
+            ],
+        ).astype('float32')
+
+        begin_pad = np.max([0, -self.context_start])
+        end_pad = np.max([0, self.context_start + self.context_length - 1])
+        total_pad = begin_pad + end_pad
+        padding_data = np.random.uniform(
+            0.1, 1, [total_pad, self.input_size[1]]
+        ).astype('float32')
+        self.pad_data = padding_data
+        self.inputs = {
+            'X': (x, self.lod),
+            'Filter': w,
+        }
+        self.inputs_val = ['X', 'Filter']
+        self.inputs_val_no_x = ['Filter']
+        self.inputs_val_no_f = ['X']
+
+        if total_pad != 0:
+            self.inputs['PaddingData'] = padding_data
+            self.inputs_val = ['X', 'PaddingData', 'Filter']
+            self.inputs_val_no_x = ['PaddingData', 'Filter']
+            self.inputs_val_no_f = ['PaddingData', 'X']
+
+        self.attrs = {
+            'contextStart': self.context_start,
+            'contextLength': self.context_length,
+            'paddingTrainable': self.padding_trainable,
+            'contextStride': self.context_stride,
+        }
+        out = seqconv(
+            x,
+            self.lod,
+            w,
+            self.context_length,
+            self.context_start,
+            self.padding_trainable,
+            self.pad_data,
+        )
+        self.outputs = {'Out': out}
+
+    def test_check_output(self):
+        # NODE(yjjiang11): This op will be deprecated.
+        self.check_output(check_dygraph=False)
+
+    def test_check_grad(self):
+        if self.padding_trainable:
+            self.check_grad(
+                set(self.inputs_val),
+                'Out',
+                max_relative_error=0.05,
+                check_dygraph=False,
+            )
+
+    def test_check_grad_input(self):
+        self.check_grad(
+            ['X'],
+            'Out',
+            max_relative_error=0.05,
+            no_grad_set=set(self.inputs_val_no_x),
+            check_dygraph=False,
+        )
+
+    def test_check_grad_padding_data(self):
+        if self.padding_trainable:
+            self.check_grad(
+                ['PaddingData'],
+                'Out',
+                no_grad_set={'X', 'Filter'},
+                check_dygraph=False,
+            )
+
+    def test_check_grad_Filter(self):
+        self.check_grad(
+            ['Filter'],
+            'Out',
+            max_relative_error=0.05,
+            no_grad_set=set(self.inputs_val_no_f),
+            check_dygraph=False,
+        )
+
+    def test_check_grad_input_filter(self):
+        if self.padding_trainable:
+            self.check_grad(
+                ['X', 'Filter'],
+                'Out',
+                max_relative_error=0.05,
+                no_grad_set={'PaddingData'},
+                check_dygraph=False,
+            )
+
+    def test_check_grad_padding_input(self):
+        if self.padding_trainable:
+            self.check_grad(
+                self.inputs_val_no_f,
+                'Out',
+                max_relative_error=0.05,
+                no_grad_set={'Filter'},
+                check_dygraph=False,
+            )
+
+    def test_check_grad_padding_filter(self):
+        if self.padding_trainable:
+            self.check_grad(
+                self.inputs_val_no_x,
+                'Out',
+                max_relative_error=0.05,
+                no_grad_set={'X'},
+                check_dygraph=False,
+            )
+
+    def init_test_case(self):
+        self.input_row = 11
+        self.context_start = 0
+        self.context_length = 1
+        self.padding_trainable = False
+        self.context_stride = 1
+
+        self.input_size = [self.input_row, 23]
+        offset_lod = [[0, 4, 5, 8, self.input_row]]
+        self.lod = [[]]
+        # convert from offset-based lod to length-based lod
+        for i in range(len(offset_lod[0]) - 1):
+            self.lod[0].append(offset_lod[0][i + 1] - offset_lod[0][i])
+        self.output_representation = 8  # output feature size
+
+
+class TestSeqProjectCase1(TestSeqProject):
+    def init_test_case(self):
+        self.input_row = 11
+        self.context_start = -1
+        self.context_length = 3
+        self.padding_trainable = True
+        self.context_stride = 1
+
+        self.input_size = [self.input_row, 50]
+        offset_lod = [[0, 4, 5, 8, self.input_row]]
+        self.lod = [[]]
+        # convert from offset-based lod to length-based lod
+        for i in range(len(offset_lod[0]) - 1):
+            self.lod[0].append(offset_lod[0][i + 1] - offset_lod[0][i])
+        self.output_representation = 8  # output feature size
+
+
+class TestSeqProjectCase2Len0(TestSeqProject):
+    def init_test_case(self):
+        self.input_row = 11
+        self.context_start = -1
+        self.context_length = 3
+        self.padding_trainable = True
+        self.context_stride = 1
+
+        self.input_size = [self.input_row, 50]
+        offset_lod = [[0, 0, 4, 5, 5, 8, self.input_row, self.input_row]]
+        self.lod = [[]]
+        # convert from offset-based lod to length-based lod
+        for i in range(len(offset_lod[0]) - 1):
+            self.lod[0].append(offset_lod[0][i + 1] - offset_lod[0][i])
+        self.output_representation = 8  # output feature size
+
+
+class TestSeqProjectCase3(TestSeqProject):
+    def init_test_case(self):
+        self.input_row = 25
+        self.context_start = 2
+        self.context_length = 3
+        self.padding_trainable = True
+        self.context_stride = 1
+
+        self.input_size = [self.input_row, 25]
+        idx = list(range(self.input_size[0]))
+        del idx[0]
+        offset_lod = [
+            [0, *np.sort(random.sample(idx, 8)).tolist(), self.input_size[0]]
+        ]
+        self.lod = [[]]
+        # convert from offset-based lod to length-based lod
+        for i in range(len(offset_lod[0]) - 1):
+            self.lod[0].append(offset_lod[0][i + 1] - offset_lod[0][i])
+        self.output_representation = 8  # output feature size
+
+
+if __name__ == '__main__':
+    unittest.main()

test/sequence/test_sequence_expand.py

@@ -0,0 +1,138 @@
+#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import unittest
+
+import numpy as np
+from op_test import OpTest
+
+
+class TestSequenceExpand(OpTest):
+    def set_data(self):
+        x_data = np.random.uniform(0.1, 1, [3, 40]).astype('float64')
+        y_data = np.random.uniform(0.1, 1, [8, 1]).astype('float64')
+        y_lod = [[1, 3, 4]]
+        self.inputs = {'X': x_data, 'Y': (y_data, y_lod)}
+
+    def compute(self):
+        x = self.inputs['X']
+        x_data, x_lod = x if type(x) == tuple else (x, None)
+        y_data, y_lod = self.inputs['Y']
+
+        if hasattr(self, 'attrs'):
+            ref_level = self.attrs['ref_level']
+        else:
+            ref_level = len(y_lod) - 1
+
+        out = np.zeros(shape=((0, *x_data.shape[1:])), dtype=x_data.dtype)
+
+        if x_lod is None:
+            # x_idx = [i for i in xrange(x_data.shape[0] + 1)]
+            x_idx = [1] * x_data.shape[0]
+        else:
+            x_idx = x_lod[0]
+            out_lod = [[]]
+
+        offset = 0
+        for i in range(len(y_lod[ref_level])):
+            repeat_num = y_lod[ref_level][i]
+            x_len = x_idx[i]
+
+            if repeat_num > 0:
+                x_sub = x_data[offset : (offset + x_len), :]
+                stacked_x_sub = x_sub
+                for r in range(repeat_num - 1):
+                    stacked_x_sub = np.vstack((stacked_x_sub, x_sub))
+                out = np.vstack((out, stacked_x_sub))
+                if x_lod is not None:
+                    for j in range(repeat_num):
+                        out_lod[0].append(x_len)
+            offset += x_len
+
+        if x_lod is None:
+            self.outputs = {'Out': out}
+        else:
+            self.outputs = {'Out': (out, out_lod)}
+
+    def setUp(self):
+        self.op_type = 'sequence_expand'
+        self.set_data()
+        self.compute()
+
+    def test_check_output(self):
+        self.check_output(check_dygraph=False)
+
+    def test_check_grad(self):
+        self.check_grad(["X"], "Out", check_dygraph=False)
+
+
+class TestSequenceExpandCase1(TestSequenceExpand):
+    def set_data(self):
+        x_data = np.random.uniform(0.1, 1, [5, 20]).astype('float64')
+        y_data = np.random.uniform(0.1, 1, [13, 1]).astype('float64')
+        y_lod = [[2, 3], [2, 2, 3, 3, 3]]
+        self.inputs = {'X': x_data, 'Y': (y_data, y_lod)}
+        self.attrs = {'ref_level': 1}
+
+
+class TestSequenceExpandCase2(TestSequenceExpand):
+    def set_data(self):
+        x_data = np.random.uniform(0.1, 1, [1, 2, 50]).astype('float64')
+        x_lod = [[1]]
+        y_data = np.random.uniform(0.1, 1, [2, 2, 2]).astype('float64')
+        y_lod = [[2], [1, 1]]
+        self.inputs = {'X': (x_data, x_lod), 'Y': (y_data, y_lod)}
+        self.attrs = {'ref_level': 0}
+
+
+class TestSequenceExpandCase3(TestSequenceExpand):
+    def set_data(self):
+        x_data = np.random.uniform(0.1, 1, [4, 25]).astype('float64')
+        x_lod = [[1, 1, 1, 1]]
+        y_data = np.random.uniform(0.1, 1, [8, 1]).astype('float64')
+        y_lod = [[2, 2, 2, 2]]
+        self.inputs = {'X': (x_data, x_lod), 'Y': (y_data, y_lod)}
+
+
+class TestSequenceExpandCase4(TestSequenceExpand):
+    def set_data(self):
+        data = np.random.uniform(0.1, 1, [5 * 20, 1])
+        x_data = np.array(data).reshape([5, 20]).astype('float64')
+        x_lod = [[2, 3]]
+        y_data = np.random.uniform(0.1, 1, [5, 1]).astype('float64')
+        y_lod = [[2], [2, 3]]
+        self.inputs = {'X': (x_data, x_lod), 'Y': (y_data, y_lod)}
+
+
+class TestSequenceExpandCase5(TestSequenceExpand):
+    def set_data(self):
+        x_data = np.random.uniform(0.1, 1, [6, 20]).astype('float64')
+        y_data = np.random.uniform(0.1, 1, [13, 1]).astype('float64')
+        y_lod = [[2, 4], [2, 2, 3, 0, 3, 3]]
+        self.inputs = {'X': x_data, 'Y': (y_data, y_lod)}
+        self.attrs = {'ref_level': 1}
+
+
+class TestSequenceExpandCase6(TestSequenceExpand):
+    def set_data(self):
+        x_data = np.random.uniform(0.1, 1, [4, 25]).astype('float64')
+        x_lod = [[1, 1, 0, 1, 1]]
+        y_data = np.random.uniform(0.1, 1, [8, 1]).astype('float64')
+        y_lod = [[0, 2, 4, 2, 0]]
+        self.inputs = {'X': (x_data, x_lod), 'Y': (y_data, y_lod)}
+
+
+if __name__ == '__main__':
+    unittest.main()

test/sequence/test_sequence_first_step.py

@@ -0,0 +1,50 @@
+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+import unittest
+
+import numpy as np
+
+import paddle
+from paddle.base.framework import Program, program_guard
+
+sys.path.append("../")
+
+
+class TestSequenceFirstStepOpError(unittest.TestCase):
+    def test_errors(self):
+        with program_guard(Program(), Program()):
+
+            def test_Variable():
+                # the input must be Variable
+                input_data = np.random.randint(1, 5, [4]).astype("int64")
+                paddle.static.nn.sequence_lod.sequence_last_step(input_data)
+
+            self.assertRaises(TypeError, test_Variable)
+
+            def test_input_dtype():
+                # the dtype of input must be int64
+                type_data = paddle.static.data(
+                    name='type_data',
+                    shape=[7, 1],
+                    dtype='int64',
+                    lod_level=1,
+                )
+                paddle.static.nn.sequence_lod.sequence_last_step(type_data)
+
+            self.assertRaises(TypeError, test_input_dtype)
+
+
+if __name__ == '__main__':
+    unittest.main()

test/sequence/test_sequence_last_step.py

@@ -0,0 +1,50 @@
+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+import unittest
+
+import numpy as np
+
+import paddle
+from paddle.base.framework import Program, program_guard
+
+sys.path.append("../")
+
+
+class TestSequenceLastStepOpError(unittest.TestCase):
+    def test_errors(self):
+        with program_guard(Program(), Program()):
+
+            def test_Variable():
+                # the input must be Variable
+                input_data = np.random.randint(1, 5, [4]).astype("int64")
+                paddle.static.nn.sequence_lod.sequence_last_step(input_data)
+
+            self.assertRaises(TypeError, test_Variable)
+
+            def test_input_dtype():
+                # the dtype of input must be int64
+                type_data = paddle.static.data(
+                    name='type_data',
+                    shape=[7, 1],
+                    dtype='int64',
+                    lod_level=1,
+                )
+                paddle.static.nn.sequence_lod.sequence_last_step(type_data)
+
+            self.assertRaises(TypeError, test_input_dtype)
+
+
+if __name__ == '__main__':
+    unittest.main()

test/sequence/test_sequence_mask.py

@@ -0,0 +1,238 @@
+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from pathlib import Path
+
+# Add test/legacy_test to sys.path
+test_dir = Path(__file__).resolve().parents[1]
+sys.path.append(str(test_dir / "legacy_test"))
+
+import unittest
+
+import numpy as np
+from op_test import OpTest
+
+import paddle
+from paddle.base.framework import (
+    convert_nptype_to_vartype,
+)
+
+
+def sequence_mask_wrapper(x, maxlen_tensor=None, maxlen=-1, mask_dtype='int64'):
+    if maxlen_tensor is not None:
+        maxlen = maxlen_tensor
+    return paddle.nn.functional.sequence_mask(
+        x, maxlen=maxlen, dtype=mask_dtype
+    )
+
+
+class SequenceMaskTestBase(OpTest):
+    def initDefaultParameters(self):
+        self.op_type = 'sequence_mask'
+        self.python_api = sequence_mask_wrapper
+        self.maxlen = 10
+        self.mask_dtype = 'int64'
+        self.x = [[0, 3, 4], [5, 7, 9]]
+
+    def initParameters(self):
+        pass
+
+    def setUp(self):
+        self.initDefaultParameters()
+        self.initParameters()
+        if not isinstance(self.x, np.ndarray):
+            self.x = np.array(self.x)
+
+        self.inputs = {'X': self.x}
+        self.outputs = {'Y': self.calc_ground_truth_mask()}
+        self.attrs = {
+            'maxlen': self.maxlen,
+            'out_dtype': convert_nptype_to_vartype(self.mask_dtype),
+        }
+
+    def calc_ground_truth_mask(self):
+        maxlen = np.max(self.x) if self.maxlen < 0 else self.maxlen
+        shape = (*self.x.shape, maxlen)
+        index_broadcast = np.broadcast_to(
+            np.reshape(range(maxlen), [1] * self.x.ndim + [-1]),
+            shape=shape,
+        )
+        x_broadcast = np.broadcast_to(
+            np.reshape(
+                self.x,
+                (*self.x.shape, -1),
+            ),
+            shape=shape,
+        )
+        return (index_broadcast < x_broadcast).astype(self.mask_dtype)
+
+    def test_check_output(self):
+        self.check_output(check_pir=True)
+
+
+class SequenceMaskTest1(SequenceMaskTestBase):
+    def initParameters(self):
+        self.mask_dtype = 'bool'
+
+
+class SequenceMaskTest2(SequenceMaskTestBase):
+    def initParameters(self):
+        self.mask_dtype = 'uint8'
+
+
+class SequenceMaskTest3(SequenceMaskTestBase):
+    def initParameters(self):
+        self.mask_dtype = 'int32'
+
+
+class SequenceMaskTest4(SequenceMaskTestBase):
+    def initParameters(self):
+        self.mask_dtype = 'float32'
+
+
+class SequenceMaskTest5(SequenceMaskTestBase):
+    def initParameters(self):
+        self.mask_dtype = 'float64'
+
+
+class SequenceMaskTest6(SequenceMaskTestBase):
+    def initParameters(self):
+        self.maxlen = -1
+
+    def test_check_output(self):
+        self.check_output(check_pir=True, check_symbol_infer=False)
+
+
+class SequenceMaskTestBase_tensor_attr(OpTest):
+    def initDefaultParameters(self):
+        self.op_type = 'sequence_mask'
+        self.python_api = sequence_mask_wrapper
+        self.maxlen = 10
+        self.maxlen_tensor = np.ones((1), 'int32') * 10
+        self.mask_dtype = 'int64'
+        self.x = [[0, 3, 4], [5, 7, 9]]
+
+    def initParameters(self):
+        pass
+
+    def setUp(self):
+        self.initDefaultParameters()
+        self.initParameters()
+        if not isinstance(self.x, np.ndarray):
+            self.x = np.array(self.x)
+
+        self.inputs = {'X': self.x, 'MaxLenTensor': self.maxlen_tensor}
+        self.outputs = {'Y': self.calc_ground_truth_mask()}
+        self.attrs = {'out_dtype': convert_nptype_to_vartype(self.mask_dtype)}
+
+    def calc_ground_truth_mask(self):
+        maxlen = np.max(self.x) if self.maxlen < 0 else self.maxlen
+        shape = (*self.x.shape, maxlen)
+        index_broadcast = np.broadcast_to(
+            np.reshape(range(maxlen), [1] * self.x.ndim + [-1]),
+            shape=shape,
+        )
+        x_broadcast = np.broadcast_to(
+            np.reshape(
+                self.x,
+                (*self.x.shape, -1),
+            ),
+            shape=shape,
+        )
+        return (index_broadcast < x_broadcast).astype(self.mask_dtype)
+
+    def test_check_output(self):
+        self.check_output(check_pir=True, check_symbol_infer=False)
+
+
+class SequenceMaskTest1_tensor_attr(SequenceMaskTestBase_tensor_attr):
+    def initParameters(self):
+        self.mask_dtype = 'bool'
+
+
+class SequenceMaskTest2_tensor_attr(SequenceMaskTestBase_tensor_attr):
+    def initParameters(self):
+        self.mask_dtype = 'uint8'
+
+
+class SequenceMaskTest3_tensor_attr(SequenceMaskTestBase_tensor_attr):
+    def initParameters(self):
+        self.mask_dtype = 'int32'
+
+
+class SequenceMaskTest4_tensor_attr(SequenceMaskTestBase_tensor_attr):
+    def initParameters(self):
+        self.mask_dtype = 'float32'
+
+
+class SequenceMaskTest5_tensor_attr(SequenceMaskTestBase_tensor_attr):
+    def initParameters(self):
+        self.mask_dtype = 'float64'
+
+
+class TestSequenceMaskOpError(unittest.TestCase):
+    def test_errors(self):
+        paddle.enable_static()
+        with paddle.static.program_guard(
+            paddle.static.Program(), paddle.static.Program()
+        ):
+            input_data = np.random.uniform(1, 5, [4]).astype("float32")
+
+            def test_Variable():
+                # the input must be Variable
+                paddle.nn.functional.sequence_mask(input_data, maxlen=4)
+
+            self.assertRaises(TypeError, test_Variable)
+        paddle.disable_static()
+
+
+class TestSequenceMaskWithEmptyTensor(unittest.TestCase):
+    def test_empty(self):
+        lengths = paddle.to_tensor(np.array([], dtype=np.int64))
+        mask = paddle.nn.functional.sequence_mask(lengths)
+        self.assertEqual(list(mask.shape), [0, 0])
+
+
+class SequenceMaskTest_ZeroSize(OpTest):
+    def initDefaultParameters(self):
+        self.op_type = 'sequence_mask'
+        self.python_api = sequence_mask_wrapper
+        self.maxlen = 10
+        self.mask_dtype = 'int64'
+        self.x = np.random.random([0, 3]).astype('int64')
+        self.y = np.random.random([0, 3, 10]).astype('int64')
+
+    def initParameters(self):
+        pass
+
+    def setUp(self):
+        self.initDefaultParameters()
+        self.initParameters()
+        if not isinstance(self.x, np.ndarray):
+            self.x = np.array(self.x)
+
+        self.inputs = {'X': self.x}
+        self.outputs = {'Y': self.y}
+        self.attrs = {
+            'maxlen': self.maxlen,
+            'out_dtype': convert_nptype_to_vartype(self.mask_dtype),
+        }
+
+    def test_check_output(self):
+        self.check_output(check_pir=True)
+
+
+if __name__ == '__main__':
+    unittest.main()

test/sequence/test_sequence_pool.py

@@ -0,0 +1,484 @@
+#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import unittest
+
+import numpy as np
+from op_test import OpTest, skip_check_grad_ci
+
+import paddle
+
+paddle.enable_static()
+
+
+def convert_to_offset(lod):
+    offset = [[0] for i in lod]
+    for i, level in enumerate(lod):
+        for seq_len in level:
+            offset[i].append(offset[i][-1] + seq_len)
+    return offset
+
+
+def compute_seqpool_sum(x, offset, out, pad_value=0.0):
+    level = len(offset) - 1
+    for i in range(len(offset[level]) - 1):
+        if offset[level][i] == offset[level][i + 1]:
+            out[i] = pad_value
+        else:
+            sub_x = x[offset[level][i] : offset[level][i + 1], :]
+            out[i] = sub_x.sum(axis=0)
+
+
+def compute_seqpool_avg(x, offset, out, pad_value=0.0):
+    level = len(offset) - 1
+    for i in range(len(offset[level]) - 1):
+        if offset[level][i] == offset[level][i + 1]:
+            out[i] = pad_value
+        else:
+            sub_x = x[offset[level][i] : offset[level][i + 1], :]
+            out[i] = sub_x.mean(axis=0)
+
+
+def compute_seqpool_sqrt(x, offset, out, pad_value=0.0):
+    level = len(offset) - 1
+    for i in range(len(offset[level]) - 1):
+        if offset[level][i] == offset[level][i + 1]:
+            out[i] = pad_value
+        else:
+            sub_x = x[offset[level][i] : offset[level][i + 1], :]
+            seq_len = offset[level][i + 1] - offset[level][i]
+            out[i] = sub_x.sum(axis=0) / np.sqrt(seq_len)
+
+
+class TestSeqAvgPool(OpTest):
+    def set_lod(self):
+        return [[11]]
+
+    def set_lod_data(self):
+        x = np.random.uniform(0.1, 1, [11, 23]).astype('float32')
+        return x
+
+    def set_data(self):
+        x = self.set_lod_data()
+        lod = self.set_lod()
+        level = len(lod) - 1
+        self.inputs = {'X': (x, lod)}
+        offset = convert_to_offset(lod)
+        out = np.zeros((len(lod[level]), x.shape[1])).astype('float32')
+        self.outputs = {'Out': out}
+        return x, lod, offset, out
+
+    def compute(self, x, offset, out):
+        self.attrs = {"pad_value": 0.0, 'pooltype': "AVERAGE"}
+        compute_seqpool_avg(x, offset, out, self.attrs["pad_value"])
+
+    def setUp(self):
+        self.op_type = 'sequence_pool'
+        x, lod, offset, out = self.set_data()
+        self.compute(x, offset, out)
+        if len(offset) > 1:
+            self.outputs = {'Out': (out, [lod[0]])}
+
+    def test_check_output(self):
+        self.check_output(check_dygraph=False)
+
+    def test_check_grad(self):
+        # Remove MaxIndex after check_grad is refined.
+        out = self.outputs['Out']
+        if isinstance(out, tuple):
+            out = out[0]
+        self.outputs['MaxIndex'] = np.zeros(out.shape).astype('int32')
+        self.check_grad(["X"], "Out", check_dygraph=False)
+
+
+class TestSeqAvgPoolBatch1(TestSeqAvgPool):
+    def set_lod(self):
+        return [[11]]
+
+    def set_lod_data(self):
+        lod = self.set_lod()
+        x, _ = self.get_sequence_batch_size_1_input(
+            lod=lod, shape=[lod[0][0], 23]
+        )
+        return x
+
+
+class TestSeqAvgPoolInstance0(TestSeqAvgPool):
+    def set_lod(self):
+        return [[0, 0, 4, 0, 3, 0, 0, 5, 0, 0]]
+
+    def set_lod_data(self):
+        lod = self.set_lod()
+        x, _ = self.get_sequence_instance_size_0_input(
+            lod=lod, shape=[sum(lod[0]), 10]
+        )
+        return x
+
+
+class TestSeqAvgPoolLen0(TestSeqAvgPool):
+    def set_lod(self):
+        return [[0, 4, 0, 7, 0]]
+
+
+class TestSeqAvgPoolLen0LoDLevel2(TestSeqAvgPool):
+    def set_lod(self):
+        return [[2, 0, 1, 2], [0, 4, 0, 7, 0]]
+
+
+class TestSeqSumPool(TestSeqAvgPool):
+    def compute(self, x, offset, out):
+        self.attrs = {"pad_value": 0.1, 'pooltype': "SUM"}
+        compute_seqpool_sum(x, offset, out, self.attrs["pad_value"])
+
+
+class TestSeqSumPoolLen0(TestSeqSumPool):
+    def set_lod(self):
+        return [[0, 4, 0, 7, 0]]
+
+
+class TestSeqSumPoolLen0LoDLevel2(TestSeqSumPool):
+    def set_lod(self):
+        return [[2, 0, 1, 2], [0, 4, 0, 7, 0]]
+
+
+class TestSeqMaxPool(TestSeqAvgPool):
+    def set_lod(self):
+        return [[13]]
+
+    def set_data(self):
+        self.op_type = 'sequence_pool'
+        x = np.random.uniform(0.1, 1, [13, 23]).astype('float32')
+        lod = self.set_lod()
+        level = len(lod) - 1
+        offset = convert_to_offset(lod)
+        for i in range(len(offset[level]) - 1):
+            l = offset[level][i + 1] - offset[level][i]
+            if l > 0:
+                x[offset[level][i] + np.random.randint(l), :] += 2.0
+
+        self.inputs = {'X': (x, lod)}
+
+        out = np.zeros((len(lod[level]), 23)).astype('float32')
+        self.outputs = {'Out': out}
+        return x, lod, offset, out
+
+    def compute(self, x, offset, out):
+        self.attrs = {"pad_value": 0.5, 'pooltype': "MAX"}
+        level = len(offset) - 1
+        for i in range(len(offset[level]) - 1):
+            if offset[level][i] == offset[level][i + 1]:
+                out[i] = self.attrs["pad_value"]
+            else:
+                sub_x = x[offset[level][i] : offset[level][i + 1], :]
+                out[i] = np.amax(sub_x, axis=0)
+
+
+class TestSeqMaxPoolLen0(TestSeqMaxPool):
+    def set_lod(self):
+        return [[0, 1, 1, 5, 6, 0]]
+
+
+class TestSeqMaxPoolLen0LoDLevel2(TestSeqMaxPool):
+    def set_lod(self):
+        return [[2, 0, 3, 1], [0, 1, 1, 5, 6, 0]]
+
+
+class TestSeqSqrtPool(TestSeqAvgPool):
+    def compute(self, x, offset, out):
+        self.attrs = {"pad_value": 0.0, 'pooltype': "SQRT"}
+        compute_seqpool_sqrt(x, offset, out, self.attrs["pad_value"])
+
+
+class TestSeqSqrtPoolLen0(TestSeqSqrtPool):
+    def set_lod(self):
+        return [[0, 7, 0, 2, 2, 0]]
+
+
+class TestSeqSqrtPoolLen0LoDLevel2(TestSeqSqrtPool):
+    def set_lod(self):
+        return [[1, 2, 0, 3], [0, 7, 0, 2, 2, 0]]
+
+
+class TestSeqLastPool(TestSeqAvgPool):
+    def compute(self, x, offset, out):
+        self.attrs = {"pad_value": 0.0, 'pooltype': "LAST"}
+        level = len(offset) - 1
+        for i in range(len(offset[level]) - 1):
+            if offset[level][i] == offset[level][i + 1]:
+                out[i] = self.attrs["pad_value"]
+            else:
+                sub_x = x[offset[level][i] : offset[level][i + 1], :]
+                out[i] = sub_x[-1, :]
+
+
+class TestSeqLastPoolLen0(TestSeqLastPool):
+    def set_lod(self):
+        return [[0, 3, 4, 0, 4, 0]]
+
+
+class TestSeqLastPoolLen0LoDLevel2(TestSeqLastPool):
+    def set_lod(self):
+        return [[1, 0, 2, 3], [0, 3, 4, 0, 4, 0]]
+
+
+class TestSeqFirstPool(TestSeqAvgPool):
+    def compute(self, x, offset, out):
+        self.attrs = {"pad_value": 0.3, 'pooltype': "FIRST"}
+        level = len(offset) - 1
+        for i in range(len(offset[level]) - 1):
+            if offset[level][i] == offset[level][i + 1]:
+                out[i] = self.attrs["pad_value"]
+            else:
+                sub_x = x[offset[level][i] : offset[level][i + 1], :]
+                out[i] = sub_x[0, :]
+
+
+class TestSeqFirstPoolLen0(TestSeqFirstPool):
+    def set_lod(self):
+        return [[0, 2, 0, 3, 6, 0]]
+
+
+class TestSeqFirstPoolLen0LoDLevel2(TestSeqFirstPool):
+    def set_lod(self):
+        return [[1, 0, 2, 3], [0, 2, 0, 3, 6, 0]]
+
+
+class TestSeqAvgPool2D(TestSeqAvgPool):
+    def set_lod(self):
+        return [[4, 1, 3, 5]]
+
+    def set_data(self):
+        self.op_type = 'sequence_pool'
+        x = np.random.uniform(0.1, 1, [13, 3, 17]).astype('float32')
+        lod = self.set_lod()
+        level = len(lod) - 1
+        self.inputs = {'X': (x, lod)}
+        offset = convert_to_offset(lod)
+
+        out = np.zeros((len(lod[level]), 3, 17)).astype('float32')
+        self.outputs = {'Out': out}
+        return x, lod, offset, out
+
+    def compute(self, x, offset, out):
+        self.attrs = {"pad_value": 0.0, 'pooltype': "AVERAGE"}
+        level = len(offset) - 1
+        for i in range(len(offset[level]) - 1):
+            if offset[level][i] == offset[level][i + 1]:
+                out[i] = self.attrs["pad_value"] * np.ones((3, 17))
+            else:
+                sub_x = np.reshape(
+                    x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 17)
+                )
+                out[i] = np.reshape(sub_x.mean(axis=0), (3, 17))
+
+
+class TestSeqAvgPool2DLen0(TestSeqAvgPool2D):
+    def set_lod(self):
+        return [[0, 5, 0, 8, 0]]
+
+
+class TestSeqAvgPool2DLen0LoDLevel2(TestSeqAvgPool2D):
+    def set_lod(self):
+        return [[1, 0, 4], [0, 5, 0, 8, 0]]
+
+
+class TestSeqSumPool2D(TestSeqAvgPool2D):
+    def compute(self, x, offset, out):
+        self.attrs = {"pad_value": 0.2, 'pooltype': "SUM"}
+        level = len(offset) - 1
+        for i in range(len(offset[level]) - 1):
+            if offset[level][i] == offset[level][i + 1]:
+                out[i] = self.attrs["pad_value"] * np.ones((3, 17))
+            else:
+                sub_x = np.reshape(
+                    x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 17)
+                )
+                out[i] = np.reshape(sub_x.sum(axis=0), (3, 17))
+
+
+class TestSeqSumPool2DLen0(TestSeqSumPool2D):
+    def set_lod(self):
+        return [[0, 8, 0, 5, 0]]
+
+
+class TestSeqSumPool2DLen0LoDLevel2(TestSeqSumPool2D):
+    def set_lod(self):
+        return [[1, 0, 4], [0, 8, 0, 5, 0]]
+
+
+class TestSeqSqrtPool2D(TestSeqAvgPool2D):
+    def compute(self, x, offset, out):
+        self.attrs = {"pad_value": 0.0, 'pooltype': "SQRT"}
+        level = len(offset) - 1
+        for i in range(len(offset[level]) - 1):
+            if offset[level][i] == offset[level][i + 1]:
+                out[i] = self.attrs["pad_value"] * np.ones((3, 17))
+            else:
+                sub_x = np.reshape(
+                    x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 17)
+                )
+                seq_len = offset[level][i + 1] - offset[level][i]
+                out[i] = np.reshape(
+                    sub_x.sum(axis=0) / np.sqrt(seq_len), (3, 17)
+                )
+
+    def test_check_grad(self):
+        # Remove MaxIndex after check_grad is refined.
+        out = self.outputs['Out']
+        if isinstance(out, tuple):
+            out = out[0]
+        self.outputs['MaxIndex'] = np.zeros(out.shape).astype('int32')
+        self.check_grad(
+            ["X"], "Out", max_relative_error=0.06, check_dygraph=False
+        )
+
+
+class TestSeqSqrtPool2DLen0(TestSeqSqrtPool2D):
+    def set_lod(self):
+        return [[0, 8, 0, 5, 0]]
+
+
+class TestSeqSqrtPool2DLen0LoDLevel2(TestSeqSqrtPool2D):
+    def set_lod(self):
+        return [[1, 0, 2, 2], [0, 8, 0, 5, 0]]
+
+
+class TestSeqMaxPool2D(TestSeqAvgPool2D):
+    def set_lod(self):
+        return [[4, 1, 3, 5]]
+
+    def set_data(self):
+        self.op_type = 'sequence_pool'
+        x = np.random.uniform(0.1, 1, [13, 3, 11]).astype('float32')
+        lod = self.set_lod()
+        level = len(lod) - 1
+        self.inputs = {'X': (x, lod)}
+        offset = convert_to_offset(lod)
+        for i in range(len(offset[level]) - 1):
+            l = offset[level][i + 1] - offset[level][i]
+            if l == 0:
+                continue
+            x[offset[level][i] + np.random.randint(l), :] += 1.0
+
+        out = np.zeros((len(lod[level]), 3, 11)).astype('float32')
+        self.outputs = {'Out': out}
+        return x, lod, offset, out
+
+    def compute(self, x, offset, out):
+        self.attrs = {"pad_value": 0.0, 'pooltype': "MAX"}
+        level = len(offset) - 1
+        for i in range(len(offset[level]) - 1):
+            if offset[level][i] == offset[level][i + 1]:
+                out[i] = self.attrs["pad_value"] * np.ones((3, 11))
+                continue
+            sub_x = np.reshape(
+                x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 11)
+            )
+            out[i] = np.reshape(np.amax(sub_x, axis=0), (3, 11))
+
+
+class TestSeqMaxPool2DLen0(TestSeqMaxPool2D):
+    def set_lod(self):
+        return [[0, 3, 0, 10, 0]]
+
+
+class TestSeqMaxPool2DLen0LoDLevel2(TestSeqMaxPool2D):
+    def set_lod(self):
+        return [[1, 0, 2, 2], [0, 3, 0, 10, 0]]
+
+
+@skip_check_grad_ci(
+    reason="Grad computation does not apply to Sequence MAX "
+    "Pool executed when is_test is true."
+)
+class TestSeqMaxPool2DInference(TestSeqMaxPool2D):
+    def compute(self, x, offset, out):
+        self.attrs = {"pad_value": 1.0, 'pooltype': "MAX", 'is_test': True}
+        level = len(offset) - 1
+        for i in range(len(offset[level]) - 1):
+            if offset[level][i] == offset[level][i + 1]:
+                out[i] = self.attrs["pad_value"] * np.ones((3, 11))
+            else:
+                sub_x = np.reshape(
+                    x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 11)
+                )
+                out[i] = np.reshape(np.amax(sub_x, axis=0), (3, 11))
+
+    def test_check_grad(self):
+        """Grad computation does not apply to Sequence MAX
+        Pool executed when is_test is true"""
+        return
+
+
+class TestSeqMaxPool2DInferenceLen0(TestSeqMaxPool2DInference):
+    def set_lod(self):
+        return [[0, 3, 0, 10, 0]]
+
+
+class TestSeqMaxPool2DInferenceLen0LoDLevel2(TestSeqMaxPool2DInference):
+    def set_lod(self):
+        return [[1, 0, 2, 2], [0, 3, 0, 10, 0]]
+
+
+class TestSeqLastPool2D(TestSeqAvgPool2D):
+    def compute(self, x, offset, out):
+        self.attrs = {"pad_value": 0.0, 'pooltype': "LAST"}
+        level = len(offset) - 1
+        for i in range(len(offset[level]) - 1):
+            if offset[level][i] == offset[level][i + 1]:
+                out[i] = self.attrs["pad_value"] * np.ones((3, 17))
+            else:
+                sub_x = np.reshape(
+                    x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 17)
+                )
+                out[i] = np.reshape(sub_x[-1, :], (3, 17))
+
+
+class TestSeqLastPool2DLen0(TestSeqLastPool2D):
+    def set_lod(self):
+        return [[0, 3, 0, 1, 9, 0]]
+
+
+class TestSeqLastPool2DLen0LoDLevel2(TestSeqLastPool2D):
+    def set_lod(self):
+        return [[1, 0, 2, 3], [0, 3, 0, 1, 9, 0]]
+
+
+class TestSeqFirstPool2D(TestSeqAvgPool2D):
+    def compute(self, x, offset, out):
+        self.attrs = {"pad_value": 0.0, 'pooltype': "FIRST"}
+        level = len(offset) - 1
+        for i in range(len(offset[level]) - 1):
+            if offset[level][i] == offset[level][i + 1]:
+                out[i] = self.attrs["pad_value"] * np.ones((3, 17))
+            else:
+                sub_x = np.reshape(
+                    x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 17)
+                )
+                out[i] = np.reshape(sub_x[0, :], (3, 17))
+
+
+class TestSeqFirstPool2DLen0(TestSeqFirstPool2D):
+    def set_lod(self):
+        return [[0, 3, 0, 3, 7, 0]]
+
+
+class TestSeqFirstPool2DLen0LoDLevel2(TestSeqFirstPool2D):
+    def set_lod(self):
+        return [[1, 0, 2, 3], [0, 3, 0, 3, 7, 0]]
+
+
+if __name__ == '__main__':
+    unittest.main()

test/sequence/test_sequence_softmax_op.py

@@ -0,0 +1,101 @@
+#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import sys
+import unittest
+
+import numpy as np
+from op_test import OpTest
+
+sys.path.append("../legacy_test")
+from test_softmax_op import stable_softmax
+
+from paddle.base import core
+
+
+class TestSequenceSoftmaxOp(OpTest):
+    def setUp(self):
+        self.op_type = "sequence_softmax"
+        self.use_cudnn = False
+        self.init_op_type()
+        self.dtype = "float32" if core.is_compiled_with_rocm() else "float64"
+        x = np.random.uniform(0.1, 1, (110, 1)).astype(self.dtype)
+        self.init_lod()
+        out = np.zeros((110, 1)).astype(self.dtype)
+        offset = 0
+        for i in range(len(self.lod[0])):
+            if self.lod[0][i] == 0:
+                continue
+            sub_x = x[offset : offset + self.lod[0][i], :]
+            sub_x = sub_x.reshape(1, self.lod[0][i])
+            sub_out = stable_softmax(sub_x)
+            out[offset : offset + self.lod[0][i], :] = sub_out.reshape(
+                self.lod[0][i], 1
+            )
+            offset += self.lod[0][i]
+
+        self.inputs = {"X": (x, self.lod)}
+        self.outputs = {"Out": out}
+        self.attrs = {
+            'use_cudnn': self.use_cudnn,
+        }
+
+    def init_lod(self):
+        self.lod = [[40, 10, 30, 30]]
+
+    def init_op_type(self):
+        pass
+
+    def test_check_output(self):
+        if self.use_cudnn:
+            place = core.CUDAPlace(0)
+            self.check_output_with_place(place, atol=1e-5, check_dygraph=False)
+        else:
+            self.check_output(check_dygraph=False)
+
+    def test_check_grad(self):
+        if self.use_cudnn:
+            place = core.CUDAPlace(0)
+            self.check_grad_with_place(place, ["X"], "Out", check_dygraph=False)
+        else:
+            self.check_grad(["X"], "Out", check_dygraph=False)
+
+
+# ----------------cudnn Sequencesoftmax----------------
+@unittest.skipIf(
+    not core.is_compiled_with_cuda(), "core is not compiled with CUDA"
+)
+class TestSequenceSoftmaxCUDNNOp(TestSequenceSoftmaxOp):
+    def init_op_type(self):
+        self.use_cudnn = True
+
+
+class TestSequenceSoftmaxOpSeqLen0Case0(TestSequenceSoftmaxOp):
+    def init_lod(self):
+        self.lod = [[40, 0, 40, 30]]
+
+
+class TestSequenceSoftmaxOpSeqLen0Case1(TestSequenceSoftmaxOp):
+    def init_lod(self):
+        self.lod = [[0, 40, 70, 0]]
+
+
+class TestSequenceSoftmaxOpSeqLen0Case2(TestSequenceSoftmaxOp):
+    def init_lod(self):
+        self.lod = [[0, 0, 0, 110]]
+
+
+if __name__ == "__main__":
+    unittest.main()