chore: import upstream snapshot with attribution

test/rnn/CMakeLists.txt

@@ -0,0 +1,14 @@
+file(
+  GLOB TEST_OPS
+  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
+  "test_*.py")
+string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
+
+foreach(TEST_OP ${TEST_OPS})
+  py_test_modules(${TEST_OP} MODULES ${TEST_OP})
+endforeach()
+
+if(NOT WIN32)
+  set_tests_properties(test_rnn_nets_static PROPERTIES TIMEOUT 120)
+  set_tests_properties(test_rnn_nets PROPERTIES TIMEOUT 120)
+endif()

test/rnn/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

test/rnn/convert.py

@@ -0,0 +1,86 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle
+
+
+def convert_params_for_cell(np_cell, paddle_cell):
+    state = np_cell.parameters
+    for k, v in paddle_cell.named_parameters():
+        v.set_value(state[k])
+
+
+def convert_params_for_cell_static(np_cell, paddle_cell, place):
+    state = np_cell.parameters
+    for k, v in paddle_cell.named_parameters():
+        scope = paddle.static.global_scope()
+        tensor = scope.find_var(v.name).get_tensor()
+        tensor.set(state[k], place)
+
+
+def convert_params_for_net(np_net, paddle_net):
+    for np_layer, paddle_layer in zip(np_net, paddle_net):
+        if hasattr(np_layer, "cell"):
+            convert_params_for_cell(np_layer.cell, paddle_layer.cell)
+        else:
+            convert_params_for_cell(np_layer.cell_fw, paddle_layer.cell_fw)
+            convert_params_for_cell(np_layer.cell_bw, paddle_layer.cell_bw)
+
+
+def convert_params_for_net_static(np_net, paddle_net, place):
+    for np_layer, paddle_layer in zip(np_net, paddle_net):
+        if hasattr(np_layer, "cell"):
+            convert_params_for_cell_static(
+                np_layer.cell, paddle_layer.cell, place
+            )
+        else:
+            convert_params_for_cell_static(
+                np_layer.cell_fw, paddle_layer.cell_fw, place
+            )
+            convert_params_for_cell_static(
+                np_layer.cell_bw, paddle_layer.cell_bw, place
+            )
+
+
+def get_params_for_cell(np_cell, num_layers, idx):
+    state = np_cell.parameters
+    weight_list = [
+        (f'{num_layers}.weight_{idx}', state['weight_ih']),
+        (f'{num_layers}.weight_{idx + 1}', state['weight_hh']),
+    ]
+    bias_list = [
+        (f'{num_layers}.bias_{idx}', state['bias_ih']),
+        (f'{num_layers}.bias_{idx + 1}', state['bias_hh']),
+    ]
+    return weight_list, bias_list
+
+
+def get_params_for_net(np_net):
+    weight_list = []
+    bias_list = []
+    for layer_idx, np_layer in enumerate(np_net):
+        if hasattr(np_layer, "cell"):
+            weight, bias = get_params_for_cell(np_layer.cell, layer_idx, 0)
+            for w, b in zip(weight, bias):
+                weight_list.append(w)
+                bias_list.append(b)
+        else:
+            for count, cell in enumerate([np_layer.cell_fw, np_layer.cell_bw]):
+                weight, bias = get_params_for_cell(cell, layer_idx, count * 2)
+                for w, b in zip(weight, bias):
+                    weight_list.append(w)
+                    bias_list.append(b)
+
+    weight_list.extend(bias_list)
+    return weight_list

test/rnn/rnn_numpy.py

@@ -0,0 +1,646 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+
+import numpy as np
+
+
+class LayerMixin:
+    def __call__(self, *args, **kwargs):
+        return self.forward(*args, **kwargs)
+
+
+class LayerListMixin(LayerMixin):
+    def __init__(self, layers=None):
+        self._layers = list(layers) if layers else []
+
+    def append(self, layer):
+        self._layers.append(layer)
+
+    def __iter__(self):
+        return iter(self._layers)
+
+
+class SimpleRNNCell(LayerMixin):
+    def __init__(
+        self,
+        input_size,
+        hidden_size,
+        weight=True,
+        bias=True,
+        nonlinearity="RNN_TANH",
+        dtype="float64",
+    ):
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.weight = weight
+        self.bias = bias
+        if nonlinearity == 'RNN_TANH':
+            self.nonlinearity = np.tanh
+        elif nonlinearity == 'RNN_RELU':
+            self.nonlinearity = lambda x: np.maximum(x, 0.0)
+
+        self.parameters = {}
+        std = 1.0 / math.sqrt(hidden_size)
+        if weight:
+            self.weight_ih = np.random.uniform(
+                -std, std, (hidden_size, input_size)
+            ).astype(dtype)
+            self.weight_hh = np.random.uniform(
+                -std, std, (hidden_size, hidden_size)
+            ).astype(dtype)
+        else:
+            self.weight_ih = np.ones((hidden_size, input_size)).astype(dtype)
+            self.weight_hh = np.ones((hidden_size, hidden_size)).astype(dtype)
+        self.parameters['weight_ih'] = self.weight_ih
+        self.parameters['weight_hh'] = self.weight_hh
+        if bias:
+            self.bias_ih = np.random.uniform(-std, std, (hidden_size,)).astype(
+                dtype
+            )
+            self.bias_hh = np.random.uniform(-std, std, (hidden_size,)).astype(
+                dtype
+            )
+        else:
+            self.bias_ih = np.zeros(hidden_size).astype(dtype)
+            self.bias_hh = np.zeros(hidden_size).astype(dtype)
+        self.parameters['bias_ih'] = self.bias_ih
+        self.parameters['bias_hh'] = self.bias_hh
+
+    def init_state(self, inputs, batch_dim_index=0):
+        batch_size = inputs.shape[batch_dim_index]
+        return np.zeros((batch_size, self.hidden_size), dtype=inputs.dtype)
+
+    def forward(self, inputs, hx=None):
+        if hx is None:
+            hx = self.init_state(inputs)
+        pre_h = hx
+        i2h = np.matmul(inputs, self.weight_ih.T)
+        if self.bias_ih is not None:
+            i2h += self.bias_ih
+        h2h = np.matmul(pre_h, self.weight_hh.T)
+        if self.bias_hh is not None:
+            h2h += self.bias_hh
+        h = self.nonlinearity(i2h + h2h)
+        return h, h
+
+
+class GRUCell(LayerMixin):
+    def __init__(
+        self, input_size, hidden_size, weight=True, bias=True, dtype="float64"
+    ):
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.weight = weight
+        self.bias = bias
+        self.parameters = {}
+        std = 1.0 / math.sqrt(hidden_size)
+        if weight:
+            self.weight_ih = np.random.uniform(
+                -std, std, (3 * hidden_size, input_size)
+            ).astype(dtype)
+            self.weight_hh = np.random.uniform(
+                -std, std, (3 * hidden_size, hidden_size)
+            ).astype(dtype)
+        else:
+            self.weight_ih = np.ones((3 * hidden_size, input_size)).astype(
+                dtype
+            )
+            self.weight_hh = np.ones((3 * hidden_size, hidden_size)).astype(
+                dtype
+            )
+        self.parameters['weight_ih'] = self.weight_ih
+        self.parameters['weight_hh'] = self.weight_hh
+        if bias:
+            self.bias_ih = np.random.uniform(
+                -std, std, (3 * hidden_size)
+            ).astype(dtype)
+            self.bias_hh = np.random.uniform(
+                -std, std, (3 * hidden_size)
+            ).astype(dtype)
+        else:
+            self.bias_ih = np.zeros(3 * hidden_size).astype(dtype)
+            self.bias_hh = np.zeros(3 * hidden_size).astype(dtype)
+        self.parameters['bias_ih'] = self.bias_ih
+        self.parameters['bias_hh'] = self.bias_hh
+
+    def init_state(self, inputs, batch_dim_index=0):
+        batch_size = inputs.shape[batch_dim_index]
+        return np.zeros((batch_size, self.hidden_size), dtype=inputs.dtype)
+
+    def forward(self, inputs, hx=None):
+        if hx is None:
+            hx = self.init_state(inputs)
+        pre_hidden = hx
+        x_gates = np.matmul(inputs, self.weight_ih.T)
+        if self.bias_ih is not None:
+            x_gates = x_gates + self.bias_ih
+        h_gates = np.matmul(pre_hidden, self.weight_hh.T)
+        if self.bias_hh is not None:
+            h_gates = h_gates + self.bias_hh
+        x_r, x_z, x_c = np.split(x_gates, 3, 1)
+        h_r, h_z, h_c = np.split(h_gates, 3, 1)
+
+        r = 1.0 / (1.0 + np.exp(-(x_r + h_r)))
+        z = 1.0 / (1.0 + np.exp(-(x_z + h_z)))
+        c = np.tanh(x_c + r * h_c)  # apply reset gate after mm
+        h = (pre_hidden - c) * z + c
+        return h, h
+
+
+class LSTMCell(LayerMixin):
+    def __init__(
+        self,
+        input_size,
+        hidden_size,
+        weight=True,
+        bias=True,
+        dtype="float64",
+        proj_size=None,
+    ):
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.weight = weight
+        self.bias = bias
+        self.parameters = {}
+        std = 1.0 / math.sqrt(hidden_size)
+        if weight:
+            self.weight_ih = np.random.uniform(
+                -std, std, (4 * hidden_size, input_size)
+            ).astype(dtype)
+            self.weight_hh = np.random.uniform(
+                -std, std, (4 * hidden_size, proj_size or hidden_size)
+            ).astype(dtype)
+        else:
+            self.weight_ih = np.ones((4 * hidden_size, input_size)).astype(
+                dtype
+            )
+            self.weight_hh = np.ones(
+                (4 * hidden_size, proj_size or hidden_size)
+            ).astype(dtype)
+
+        self.parameters['weight_ih'] = self.weight_ih
+        self.parameters['weight_hh'] = self.weight_hh
+
+        self.proj_size = proj_size
+        if proj_size:
+            self.weight_ho = np.random.uniform(
+                -std, std, (hidden_size, proj_size)
+            ).astype(dtype)
+            self.parameters['weight_ho'] = self.weight_ho
+
+        if bias:
+            self.bias_ih = np.random.uniform(
+                -std, std, (4 * hidden_size)
+            ).astype(dtype)
+            self.bias_hh = np.random.uniform(
+                -std, std, (4 * hidden_size)
+            ).astype(dtype)
+        else:
+            self.bias_ih = np.zeros(4 * hidden_size).astype(dtype)
+            self.bias_hh = np.zeros(4 * hidden_size).astype(dtype)
+        self.parameters['bias_ih'] = self.bias_ih
+        self.parameters['bias_hh'] = self.bias_hh
+
+    def init_state(self, inputs, batch_dim_index=0):
+        batch_size = inputs.shape[batch_dim_index]
+        init_h = np.zeros((batch_size, self.hidden_size), dtype=inputs.dtype)
+        init_c = np.zeros((batch_size, self.hidden_size), dtype=inputs.dtype)
+        return init_h, init_c
+
+    def forward(self, inputs, hx=None):
+        if hx is None:
+            hx = self.init_state(inputs)
+        pre_hidden, pre_cell = hx
+        gates = np.matmul(inputs, self.weight_ih.T)
+        if self.bias_ih is not None:
+            gates = gates + self.bias_ih
+        gates += np.matmul(pre_hidden, self.weight_hh.T)
+        if self.bias_hh is not None:
+            gates = gates + self.bias_hh
+
+        chunked_gates = np.split(gates, 4, -1)
+
+        i = 1.0 / (1.0 + np.exp(-chunked_gates[0]))
+        f = 1.0 / (1.0 + np.exp(-chunked_gates[1]))
+        o = 1.0 / (1.0 + np.exp(-chunked_gates[3]))
+        c = f * pre_cell + i * np.tanh(chunked_gates[2])
+        h = o * np.tanh(c)
+
+        if self.proj_size:
+            h = np.matmul(h, self.weight_ho)
+
+        return h, (h, c)
+
+
+def sequence_mask(lengths, max_len=None):
+    if max_len is None:
+        max_len = np.max(lengths)
+    else:
+        assert max_len >= np.max(lengths)
+    return np.arange(max_len) < np.expand_dims(lengths, -1)
+
+
+def update_state(mask, new, old):
+    if not isinstance(old, (tuple, list)):
+        return np.where(mask, new, old)
+    else:
+        return tuple(np.where(mask, x, y) for x, y in zip(new, old))
+
+
+def rnn(
+    cell,
+    inputs,
+    initial_states,
+    sequence_length=None,
+    time_major=False,
+    is_reverse=False,
+):
+    if not time_major:
+        inputs = np.transpose(inputs, [1, 0, 2])
+    if is_reverse:
+        inputs = np.flip(inputs, 0)
+
+    if initial_states is None:
+        initial_states = cell.init_state(inputs, 1)
+
+    if sequence_length is None:
+        mask = None
+    else:
+        mask = np.transpose(sequence_mask(sequence_length), [1, 0])
+        mask = np.expand_dims(mask, -1)
+        if is_reverse:
+            mask = np.flip(mask, 0)
+
+    time_steps = inputs.shape[0]
+    state = initial_states
+    outputs = []
+    for t in range(time_steps):
+        x_t = inputs[t]
+        if mask is not None:
+            m_t = mask[t]
+            y, new_state = cell(x_t, state)
+            y = np.where(m_t, y, 0.0)
+            outputs.append(y)
+            state = update_state(m_t, new_state, state)
+        else:
+            y, new_state = cell(x_t, state)
+            outputs.append(y)
+            state = new_state
+
+    outputs = np.stack(outputs)
+    final_state = state
+
+    if is_reverse:
+        outputs = np.flip(outputs, 0)
+    if not time_major:
+        outputs = np.transpose(outputs, [1, 0, 2])
+    return outputs, final_state
+
+
+def birnn(
+    cell_fw,
+    cell_bw,
+    inputs,
+    initial_states,
+    sequence_length=None,
+    time_major=False,
+):
+    states_fw, states_bw = initial_states
+    outputs_fw, states_fw = rnn(
+        cell_fw, inputs, states_fw, sequence_length, time_major=time_major
+    )
+
+    outputs_bw, states_bw = rnn(
+        cell_bw,
+        inputs,
+        states_bw,
+        sequence_length,
+        time_major=time_major,
+        is_reverse=True,
+    )
+
+    outputs = np.concatenate((outputs_fw, outputs_bw), -1)
+    final_states = (states_fw, states_bw)
+    return outputs, final_states
+
+
+def flatten(nested):
+    return list(_flatten(nested))
+
+
+def _flatten(nested):
+    for item in nested:
+        if isinstance(item, (list, tuple)):
+            yield from _flatten(item)
+        else:
+            yield item
+
+
+def unstack(array, axis=0):
+    num = array.shape[axis]
+    sub_arrays = np.split(array, num, axis)
+    return [np.squeeze(sub_array, axis) for sub_array in sub_arrays]
+
+
+def dropout(array, p=0.5):
+    if p == 0.0:
+        return array
+    mask = (np.random.uniform(size=array.shape) < (1 - p)).astype(array.dtype)
+    return array * (mask / (1 - p))
+
+
+def split_states(states, bidirectional=False, state_components=1):
+    if state_components == 1:
+        states = unstack(states)
+        if not bidirectional:
+            return states
+        else:
+            return list(zip(states[::2], states[1::2]))
+    else:
+        assert len(states) == state_components
+        states = tuple([unstack(item) for item in states])
+        if not bidirectional:
+            return list(zip(*states))
+        else:
+            states = list(zip(*states))
+            return list(zip(states[::2], states[1::2]))
+
+
+def concat_states(states, bidirectional=False, state_components=1):
+    if state_components == 1:
+        return np.stack(flatten(states))
+    else:
+        states = flatten(states)
+        components = []
+        for i in range(state_components):
+            components.append(states[i::state_components])
+        return [np.stack(item) for item in components]
+
+
+class RNN(LayerMixin):
+    def __init__(self, cell, is_reverse=False, time_major=False):
+        super().__init__()
+        self.cell = cell
+        if not hasattr(self.cell, "call"):
+            # for non-dygraph mode, `rnn` api uses cell.call
+            self.cell.call = self.cell.forward
+        self.is_reverse = is_reverse
+        self.time_major = time_major
+
+    def forward(self, inputs, initial_states=None, sequence_length=None):
+        final_outputs, final_states = rnn(
+            self.cell,
+            inputs,
+            initial_states=initial_states,
+            sequence_length=sequence_length,
+            time_major=self.time_major,
+            is_reverse=self.is_reverse,
+        )
+        return final_outputs, final_states
+
+
+class BiRNN(LayerMixin):
+    def __init__(self, cell_fw, cell_bw, time_major=False):
+        super().__init__()
+        self.cell_fw = cell_fw
+        self.cell_bw = cell_bw
+        self.time_major = time_major
+
+    def forward(
+        self, inputs, initial_states=None, sequence_length=None, **kwargs
+    ):
+        if isinstance(initial_states, (list, tuple)):
+            assert len(initial_states) == 2, (
+                "length of initial_states should be 2 when it is a list/tuple"
+            )
+        else:
+            initial_states = [initial_states, initial_states]
+
+        outputs, final_states = birnn(
+            self.cell_fw,
+            self.cell_bw,
+            inputs,
+            initial_states,
+            sequence_length,
+            self.time_major,
+        )
+        return outputs, final_states
+
+
+class RNNMixin(LayerListMixin):
+    def forward(self, inputs, initial_states=None, sequence_length=None):
+        batch_index = 1 if self.time_major else 0
+        batch_size = inputs.shape[batch_index]
+        dtype = inputs.dtype
+        if initial_states is None:
+            state_shape = (self.num_layers * self.num_directions, batch_size)
+            proj_size = self.proj_size if hasattr(self, 'proj_size') else None
+            dims = ((proj_size or self.hidden_size,), (self.hidden_size,))
+            if self.state_components == 1:
+                initial_states = np.zeros(state_shape + dims[0], dtype)
+            else:
+                initial_states = tuple(
+                    [
+                        np.zeros(state_shape + dims[i], dtype)
+                        for i in range(self.state_components)
+                    ]
+                )
+        states = split_states(
+            initial_states, self.num_directions == 2, self.state_components
+        )
+        final_states = []
+        input_temp = inputs
+        for i, rnn_layer in enumerate(self):
+            if i > 0:
+                input_temp = dropout(inputs, self.dropout)
+            outputs, final_state = rnn_layer(
+                input_temp, states[i], sequence_length
+            )
+            final_states.append(final_state)
+            inputs = outputs
+
+        final_states = concat_states(
+            final_states, self.num_directions == 2, self.state_components
+        )
+        return outputs, final_states
+
+
+class SimpleRNN(RNNMixin):
+    def __init__(
+        self,
+        input_size,
+        hidden_size,
+        num_layers=1,
+        nonlinearity="RNN_TANH",
+        direction="forward",
+        dropout=0.0,
+        time_major=False,
+        dtype="float64",
+    ):
+        super().__init__()
+        bidirectional_list = ["bidirectional", "bidirect"]
+        if direction in ["forward"]:
+            is_reverse = False
+            cell = SimpleRNNCell(
+                input_size, hidden_size, nonlinearity=nonlinearity, dtype=dtype
+            )
+            self.append(RNN(cell, is_reverse, time_major))
+            for i in range(1, num_layers):
+                cell = SimpleRNNCell(
+                    hidden_size,
+                    hidden_size,
+                    nonlinearity=nonlinearity,
+                    dtype=dtype,
+                )
+                self.append(RNN(cell, is_reverse, time_major))
+        elif direction in bidirectional_list:
+            cell_fw = SimpleRNNCell(
+                input_size, hidden_size, nonlinearity=nonlinearity, dtype=dtype
+            )
+            cell_bw = SimpleRNNCell(
+                input_size, hidden_size, nonlinearity=nonlinearity, dtype=dtype
+            )
+            self.append(BiRNN(cell_fw, cell_bw, time_major))
+            for i in range(1, num_layers):
+                cell_fw = SimpleRNNCell(
+                    2 * hidden_size,
+                    hidden_size,
+                    nonlinearity=nonlinearity,
+                    dtype=dtype,
+                )
+                cell_bw = SimpleRNNCell(
+                    2 * hidden_size,
+                    hidden_size,
+                    nonlinearity=nonlinearity,
+                    dtype=dtype,
+                )
+                self.append(BiRNN(cell_fw, cell_bw, time_major))
+        else:
+            raise ValueError(
+                "direction should be forward, backward or bidirectional, "
+                f"received direction = {direction}"
+            )
+
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.dropout = dropout
+        self.num_directions = 2 if direction in bidirectional_list else 1
+        self.time_major = time_major
+        self.num_layers = num_layers
+        self.state_components = 1
+
+
+class LSTM(RNNMixin):
+    def __init__(
+        self,
+        input_size,
+        hidden_size,
+        num_layers=1,
+        direction="forward",
+        dropout=0.0,
+        time_major=False,
+        dtype="float64",
+        proj_size=None,
+    ):
+        super().__init__()
+
+        bidirectional_list = ["bidirectional", "bidirect"]
+        in_size = proj_size or hidden_size
+        if direction in ["forward"]:
+            is_reverse = False
+            cell = LSTMCell(
+                input_size, hidden_size, dtype=dtype, proj_size=proj_size
+            )
+            self.append(RNN(cell, is_reverse, time_major))
+            for i in range(1, num_layers):
+                cell = LSTMCell(
+                    in_size, hidden_size, dtype=dtype, proj_size=proj_size
+                )
+                self.append(RNN(cell, is_reverse, time_major))
+        elif direction in bidirectional_list:
+            cell_fw = LSTMCell(
+                input_size, hidden_size, dtype=dtype, proj_size=proj_size
+            )
+            cell_bw = LSTMCell(
+                input_size, hidden_size, dtype=dtype, proj_size=proj_size
+            )
+            self.append(BiRNN(cell_fw, cell_bw, time_major))
+            for i in range(1, num_layers):
+                cell_fw = LSTMCell(
+                    2 * in_size, hidden_size, dtype=dtype, proj_size=proj_size
+                )
+                cell_bw = LSTMCell(
+                    2 * in_size, hidden_size, dtype=dtype, proj_size=proj_size
+                )
+                self.append(BiRNN(cell_fw, cell_bw, time_major))
+        else:
+            raise ValueError(
+                "direction should be forward, backward or bidirectional, "
+                f"received direction = {direction}"
+            )
+
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.dropout = dropout
+        self.num_directions = 2 if direction in bidirectional_list else 1
+        self.time_major = time_major
+        self.num_layers = num_layers
+        self.state_components = 2
+        self.proj_size = proj_size
+
+
+class GRU(RNNMixin):
+    def __init__(
+        self,
+        input_size,
+        hidden_size,
+        num_layers=1,
+        direction="forward",
+        dropout=0.0,
+        time_major=False,
+        dtype="float64",
+    ):
+        super().__init__()
+
+        bidirectional_list = ["bidirectional", "bidirect"]
+        if direction in ["forward"]:
+            is_reverse = False
+            cell = GRUCell(input_size, hidden_size, dtype=dtype)
+            self.append(RNN(cell, is_reverse, time_major))
+            for i in range(1, num_layers):
+                cell = GRUCell(hidden_size, hidden_size, dtype=dtype)
+                self.append(RNN(cell, is_reverse, time_major))
+        elif direction in bidirectional_list:
+            cell_fw = GRUCell(input_size, hidden_size, dtype=dtype)
+            cell_bw = GRUCell(input_size, hidden_size, dtype=dtype)
+            self.append(BiRNN(cell_fw, cell_bw, time_major))
+            for i in range(1, num_layers):
+                cell_fw = GRUCell(2 * hidden_size, hidden_size, dtype=dtype)
+                cell_bw = GRUCell(2 * hidden_size, hidden_size, dtype=dtype)
+                self.append(BiRNN(cell_fw, cell_bw, time_major))
+        else:
+            raise ValueError(
+                "direction should be forward, backward or bidirectional, "
+                f"received direction = {direction}"
+            )
+
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.dropout = dropout
+        self.num_directions = 2 if direction in bidirectional_list else 1
+        self.time_major = time_major
+        self.num_layers = num_layers
+        self.state_components = 1

test/rnn/test_rnn_api.py

@@ -0,0 +1,403 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import paddle
+
+paddle.set_default_dtype("float64")
+
+import unittest
+
+import numpy as np
+
+from paddle import base
+
+paddle.enable_static()
+
+bidirectional_list = ["bidirectional", "bidirect"]
+
+
+class TestSimpleRNN(unittest.TestCase):
+    def __init__(self, time_major=True, direction="forward", place="cpu"):
+        super().__init__("runTest")
+        self.time_major = time_major
+        self.direction = direction
+        self.num_directions = 2 if direction in bidirectional_list else 1
+        self.place = place
+        self.batch_size = 4
+        self.input_size = 16
+        self.hidden_size = 16
+        self.seq_len = 12
+        self.seed = 1234
+
+    def setUp(self):
+        # Since `set_device` is global, set `set_device` in `setUp` rather than
+        # `__init__` to avoid using an error device set by another test case.
+
+        place = paddle.set_device(self.place)
+        paddle.disable_static(self.place)
+        paddle.seed(self.seed)
+        if paddle.framework.use_pir_api():
+            with paddle.pir_utils.OldIrGuard():
+                # Note: dygraph use self.main_program.global_block().create_parameter(), it's need manual seed to old Program
+                paddle.framework.random._manual_program_seed(self.seed)
+            paddle.framework.random._manual_program_seed(self.seed)
+        else:
+            paddle.framework.random._manual_program_seed(self.seed)
+        cell_dy = paddle.nn.SimpleRNNCell(self.input_size, self.hidden_size)
+        self.rnn_net = paddle.nn.RNN(cell_dy, time_major=self.time_major)
+
+        paddle.enable_static()
+
+        with paddle.base.unique_name.guard():
+            main_program = paddle.static.Program()
+            startup_program = paddle.static.Program()
+            with paddle.static.program_guard(
+                main_program=main_program, startup_program=startup_program
+            ):
+                paddle.seed(self.seed)
+                paddle.framework.random._manual_program_seed(self.seed)
+
+                self.exe = base.Executor(
+                    base.CPUPlace()
+                    if self.place == "cpu"
+                    else base.CUDAPlace(0)
+                )
+
+                rnn_in_data = paddle.static.data(
+                    "x",
+                    [None, self.batch_size, self.hidden_size],
+                    dtype="float64",
+                )
+                pre_h_data = paddle.static.data(
+                    "pre_h",
+                    [self.batch_size, self.hidden_size],
+                    dtype="float64",
+                )
+                seq_len_data = paddle.static.data(
+                    "seq_len", [self.batch_size], dtype="int64"
+                )
+                cell_st = paddle.nn.SimpleRNNCell(
+                    self.input_size, self.hidden_size
+                )
+                self.rnn_st = paddle.nn.RNN(cell_st, time_major=self.time_major)
+                st_out, st_last_h = self.rnn_st(
+                    rnn_in_data, pre_h_data, sequence_length=seq_len_data
+                )
+
+                self.fetch_list = [st_out, st_last_h]
+
+                self.exe.run(paddle.static.default_startup_program())
+
+                self.main_program = paddle.static.default_main_program()
+
+            paddle.disable_static(self.place)
+
+    def test_base(self, test_seq_len=False):
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+        prev_h = np.random.randn(4, 16)
+
+        paddle.disable_static(self.place)
+        if test_seq_len:
+            seq_len = np.array([9, 10, 8, 12], "int64")
+        else:
+            seq_len = np.array([12, 12, 12, 12], "int64")
+
+        y1, h1 = self.rnn_net(
+            paddle.to_tensor(x),
+            paddle.to_tensor(prev_h),
+            sequence_length=paddle.to_tensor(seq_len),
+        )
+
+        paddle.enable_static()
+        out = self.exe.run(
+            self.main_program,
+            feed={"x": x, "pre_h": prev_h, "seq_len": seq_len},
+            fetch_list=[self.fetch_list],
+        )
+
+        y2, h2 = out
+
+        np.testing.assert_allclose(y1.numpy(), y2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1.numpy(), h2, atol=1e-8, rtol=1e-5)
+
+    def runTest(self):
+        self.test_base()
+        self.test_base(True)
+
+
+class TestGRU(unittest.TestCase):
+    def __init__(self, time_major=True, direction="forward", place="cpu"):
+        super().__init__("runTest")
+        self.time_major = time_major
+        self.direction = direction
+        self.num_directions = 2 if direction in bidirectional_list else 1
+        self.place = place
+        self.batch_size = 4
+        self.input_size = 16
+        self.hidden_size = 16
+        self.seq_len = 12
+        self.seed = 1234
+
+    def setUp(self):
+        # Since `set_device` is global, set `set_device` in `setUp` rather than
+        # `__init__` to avoid using an error device set by another test case.
+
+        place = paddle.set_device(self.place)
+        paddle.disable_static(self.place)
+        paddle.seed(self.seed)
+        if paddle.framework.use_pir_api():
+            with paddle.pir_utils.OldIrGuard():
+                # Note: dygraph use self.main_program.global_block().create_parameter(), it's need manual seed to old Program
+                paddle.framework.random._manual_program_seed(self.seed)
+            paddle.framework.random._manual_program_seed(self.seed)
+        else:
+            paddle.framework.random._manual_program_seed(self.seed)
+        cell_dy = paddle.nn.GRUCell(self.input_size, self.hidden_size)
+        self.rnn_net = paddle.nn.RNN(cell_dy, time_major=self.time_major)
+
+        paddle.enable_static()
+
+        with paddle.base.unique_name.guard():
+            main_program = paddle.static.Program()
+            startup_program = paddle.static.Program()
+            with paddle.static.program_guard(
+                main_program=main_program, startup_program=startup_program
+            ):
+                paddle.seed(self.seed)
+                paddle.framework.random._manual_program_seed(self.seed)
+
+                self.exe = base.Executor(
+                    base.CPUPlace()
+                    if self.place == "cpu"
+                    else base.CUDAPlace(0)
+                )
+
+                rnn_in_data = paddle.static.data(
+                    "x",
+                    [None, self.batch_size, self.hidden_size],
+                    dtype="float64",
+                )
+                pre_h_data = paddle.static.data(
+                    "pre_h",
+                    [self.batch_size, self.hidden_size],
+                    dtype="float64",
+                )
+                seq_len_data = paddle.static.data(
+                    "seq_len", [self.batch_size], dtype="int64"
+                )
+                cell_st = paddle.nn.GRUCell(self.input_size, self.hidden_size)
+                self.rnn_st = paddle.nn.RNN(cell_st, time_major=self.time_major)
+                st_out, st_last_h = self.rnn_st(
+                    rnn_in_data, pre_h_data, sequence_length=seq_len_data
+                )
+
+                self.fetch_list = [st_out, st_last_h]
+
+                self.exe.run(paddle.static.default_startup_program())
+
+                self.main_program = paddle.static.default_main_program()
+
+            paddle.disable_static(self.place)
+
+    def test_base(self, test_seq_len=False):
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+        prev_h = np.random.randn(4, 16)
+
+        paddle.disable_static(self.place)
+        if test_seq_len:
+            seq_len = np.array([9, 10, 8, 12], "int64")
+        else:
+            seq_len = np.array([12, 12, 12, 12], "int64")
+
+        y1, h1 = self.rnn_net(
+            paddle.to_tensor(x),
+            paddle.to_tensor(prev_h),
+            sequence_length=paddle.to_tensor(seq_len),
+        )
+
+        paddle.enable_static()
+        out = self.exe.run(
+            self.main_program,
+            feed={"x": x, "pre_h": prev_h, "seq_len": seq_len},
+            fetch_list=[self.fetch_list],
+        )
+
+        y2, h2 = out
+
+        np.testing.assert_allclose(y1.numpy(), y2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1.numpy(), h2, atol=1e-8, rtol=1e-5)
+
+    def runTest(self):
+        self.test_base()
+        self.test_base(True)
+
+
+class TestGRUBackward(unittest.TestCase):
+    def __init__(self, time_major=True, direction="forward", place="cpu"):
+        super().__init__("runTest")
+        self.time_major = time_major
+        self.direction = direction
+        self.num_directions = 2 if direction in bidirectional_list else 1
+        self.place = place
+        self.batch_size = 4
+        self.input_size = 4
+        self.hidden_size = 4
+        self.seq_len = 12
+        self.seed = 1234
+
+    def setUp(self):
+        # Since `set_device` is global, set `set_device` in `setUp` rather than
+        # `__init__` to avoid using an error device set by another test case.
+
+        place = paddle.set_device(self.place)
+        paddle.disable_static(self.place)
+        paddle.seed(self.seed)
+        if paddle.framework.use_pir_api():
+            with paddle.pir_utils.OldIrGuard():
+                # Note: dygraph use self.main_program.global_block().create_parameter(), it's need manual seed to old Program
+                paddle.framework.random._manual_program_seed(self.seed)
+            paddle.framework.random._manual_program_seed(self.seed)
+        else:
+            paddle.framework.random._manual_program_seed(self.seed)
+        cell_dy = paddle.nn.SimpleRNNCell(self.input_size, self.hidden_size)
+        self.rnn_net = paddle.nn.RNN(cell_dy, time_major=self.time_major)
+
+        paddle.enable_static()
+
+        with paddle.base.unique_name.guard():
+            main_program = paddle.static.Program()
+            startup_program = paddle.static.Program()
+            with paddle.static.program_guard(
+                main_program=main_program, startup_program=startup_program
+            ):
+                paddle.seed(self.seed)
+                paddle.framework.random._manual_program_seed(self.seed)
+
+                self.exe = paddle.base.Executor(
+                    base.CPUPlace()
+                    if self.place == "cpu"
+                    else base.CUDAPlace(0)
+                )
+
+                rnn_in_data = paddle.static.data(
+                    "x",
+                    [None, self.batch_size, self.hidden_size],
+                    dtype="float64",
+                )
+                pre_h_data = paddle.static.data(
+                    "pre_h",
+                    [self.batch_size, self.hidden_size],
+                    dtype="float64",
+                )
+                seq_len_data = paddle.static.data(
+                    "seq_len", [self.batch_size], dtype="int64"
+                )
+                pre_h_data.stop_gradient = False
+                rnn_in_data.stop_gradient = False
+
+                cell_st = paddle.nn.SimpleRNNCell(
+                    self.input_size, self.hidden_size
+                )
+                self.rnn_st = paddle.nn.RNN(cell_st, time_major=self.time_major)
+
+                st_out, st_last_h = self.rnn_st(
+                    rnn_in_data, pre_h_data, sequence_length=seq_len_data
+                )
+                loss = paddle.sum(st_out)
+                sgd = paddle.optimizer.SGD(0.0)
+
+                if paddle.framework.in_pir_mode():
+                    rnn_in_data.persistable = True
+                    pre_h_data.persistable = True
+                    params_grads = paddle.base.backward.append_backward(loss)
+                    pre_h_data_grad = None
+                    rnn_in_data_grad = None
+                    for p, g in params_grads:
+                        if p.is_same(rnn_in_data):
+                            rnn_in_data_grad = g
+                        elif p.is_same(pre_h_data):
+                            pre_h_data_grad = g
+
+                    self.fetch_list = [
+                        st_out,
+                        st_last_h,
+                        pre_h_data_grad,
+                        rnn_in_data_grad,
+                    ]
+                else:
+                    sgd.minimize(loss)
+                    self.fetch_list = [
+                        st_out,
+                        st_last_h,
+                        "pre_h@GRAD",
+                        "x@GRAD",
+                    ]
+
+                self.exe.run(paddle.static.default_startup_program())
+
+                self.main_program = paddle.static.default_main_program()
+
+            paddle.disable_static(self.place)
+
+    def test_base(self, test_seq_len=False):
+        x = np.random.randn(12, 4, self.hidden_size)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+        prev_h = np.random.randn(4, self.hidden_size)
+
+        paddle.disable_static(self.place)
+        if test_seq_len:
+            seq_len = np.array([9, 10, 8, 12], "int64")
+        else:
+            seq_len = np.array([12, 12, 12, 12], "int64")
+
+        x_in = paddle.to_tensor(x)
+        h_in = paddle.to_tensor(prev_h)
+        x_in.stop_gradient = False
+        h_in.stop_gradient = False
+        y1, h1 = self.rnn_net(
+            x_in,
+            h_in,
+            sequence_length=paddle.to_tensor(seq_len),
+        )
+        loss = y1.sum()
+        loss.backward()
+
+        h1_grad = h_in.gradient()
+
+        paddle.enable_static()
+        out = self.exe.run(
+            self.main_program,
+            feed={"x": x, "pre_h": prev_h, "seq_len": seq_len},
+            fetch_list=[self.fetch_list],
+        )
+
+        y2, h2, g1, g2 = out
+
+        np.testing.assert_allclose(h1_grad, g1, atol=1e-8, rtol=1e-5)
+        self.exe._executor_cache._get_cached_program_and_executor_pir_mode.cache_clear()
+
+    def runTest(self):
+        self.test_base(True)
+        self.test_base()
+
+
+if __name__ == "__main__":
+    paddle.enable_static()
+    unittest.main()

test/rnn/test_rnn_cells.py

@@ -0,0 +1,221 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle
+
+paddle.framework.set_default_dtype("float64")
+
+import unittest
+
+import numpy as np
+from convert import convert_params_for_cell
+from rnn_numpy import GRUCell, LSTMCell, SimpleRNNCell
+
+
+class TestSimpleRNNCell(unittest.TestCase):
+    def __init__(self, weight=True, bias=True, place="cpu"):
+        super().__init__(methodName="runTest")
+        self.weight = weight
+        self.bias = bias
+        self.place = (
+            paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0)
+        )
+
+    def setUp(self):
+        paddle.disable_static(self.place)
+        rnn1 = SimpleRNNCell(16, 32, weight=self.weight, bias=self.bias)
+        rnn2 = paddle.nn.SimpleRNNCell(
+            16,
+            32,
+            weight_ih_attr=self.weight,
+            weight_hh_attr=self.weight,
+            bias_ih_attr=self.bias,
+            bias_hh_attr=self.bias,
+        )
+        convert_params_for_cell(rnn1, rnn2)
+
+        self.rnn1 = rnn1
+        self.rnn2 = rnn2
+
+    def test_with_initial_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(4, 16)
+        prev_h = np.random.randn(4, 32)
+
+        y1, h1 = rnn1(x, prev_h)
+        y2, h2 = rnn2(paddle.to_tensor(x), paddle.to_tensor(prev_h))
+        np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_with_zero_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(4, 16)
+
+        y1, h1 = rnn1(x)
+        y2, h2 = rnn2(paddle.to_tensor(x))
+        np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_errors(self):
+        def test_zero_hidden_size():
+            cell = paddle.nn.SimpleRNNCell(-1, 0)
+
+        self.assertRaises(ValueError, test_zero_hidden_size)
+
+    def runTest(self):
+        self.test_with_initial_state()
+        self.test_with_zero_state()
+        self.test_errors()
+
+
+class TestGRUCell(unittest.TestCase):
+    def __init__(self, weight=True, bias=True, place="cpu"):
+        super().__init__(methodName="runTest")
+        self.weight = weight
+        self.bias = bias
+        self.place = (
+            paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0)
+        )
+
+    def setUp(self):
+        paddle.disable_static(self.place)
+        rnn1 = GRUCell(16, 32, weight=self.weight, bias=self.bias)
+        rnn2 = paddle.nn.GRUCell(
+            16,
+            32,
+            weight_ih_attr=self.weight,
+            weight_hh_attr=self.weight,
+            bias_ih_attr=self.bias,
+            bias_hh_attr=self.bias,
+        )
+        convert_params_for_cell(rnn1, rnn2)
+
+        self.rnn1 = rnn1
+        self.rnn2 = rnn2
+
+    def test_with_initial_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(4, 16)
+        prev_h = np.random.randn(4, 32)
+
+        y1, h1 = rnn1(x, prev_h)
+        y2, h2 = rnn2(paddle.to_tensor(x), paddle.to_tensor(prev_h))
+        np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_with_zero_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(4, 16)
+
+        y1, h1 = rnn1(x)
+        y2, h2 = rnn2(paddle.to_tensor(x))
+        np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_errors(self):
+        def test_zero_hidden_size():
+            cell = paddle.nn.GRUCell(-1, 0)
+
+        self.assertRaises(ValueError, test_zero_hidden_size)
+
+    def runTest(self):
+        self.test_with_initial_state()
+        self.test_with_zero_state()
+        self.test_errors()
+
+
+class TestLSTMCell(unittest.TestCase):
+    def __init__(self, weight=True, bias=True, place="cpu"):
+        super().__init__(methodName="runTest")
+        self.weight = weight
+        self.bias = bias
+        self.place = (
+            paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0)
+        )
+
+    def setUp(self):
+        rnn1 = LSTMCell(16, 32, weight=self.weight, bias=self.bias)
+        rnn2 = paddle.nn.LSTMCell(
+            16,
+            32,
+            weight_ih_attr=self.weight,
+            weight_hh_attr=self.weight,
+            bias_ih_attr=self.bias,
+            bias_hh_attr=self.bias,
+        )
+        convert_params_for_cell(rnn1, rnn2)
+
+        self.rnn1 = rnn1
+        self.rnn2 = rnn2
+
+    def test_with_initial_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(4, 16)
+        prev_h = np.random.randn(4, 32)
+        prev_c = np.random.randn(4, 32)
+
+        y1, (h1, c1) = rnn1(x, (prev_h, prev_c))
+        y2, (h2, c2) = rnn2(
+            paddle.to_tensor(x),
+            (paddle.to_tensor(prev_h), paddle.to_tensor(prev_c)),
+        )
+        np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(c1, c2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_with_zero_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(4, 16)
+
+        y1, (h1, c1) = rnn1(x)
+        y2, (h2, c2) = rnn2(paddle.to_tensor(x))
+        np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(c1, c2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_errors(self):
+        def test_zero_hidden_size():
+            cell = paddle.nn.LSTMCell(-1, 0)
+
+        self.assertRaises(ValueError, test_zero_hidden_size)
+
+    def runTest(self):
+        self.test_with_initial_state()
+        self.test_with_zero_state()
+        self.test_errors()
+
+
+def load_tests(loader, tests, pattern):
+    suite = unittest.TestSuite()
+    devices = ["cpu", "gpu"] if paddle.base.is_compiled_with_cuda() else ["cpu"]
+    for weight in [True, False]:
+        for bias in [True, False]:
+            for device in devices:
+                for test_class in [
+                    TestSimpleRNNCell,
+                    TestGRUCell,
+                    TestLSTMCell,
+                ]:
+                    suite.addTest(test_class(weight, bias, device))
+    return suite
+
+
+if __name__ == "__main__":
+    unittest.main()

test/rnn/test_rnn_cells_static.py

@@ -0,0 +1,362 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle
+
+paddle.framework.set_default_dtype("float64")
+paddle.enable_static()
+
+import sys
+import unittest
+
+import numpy as np
+from convert import convert_params_for_cell_static
+
+sys.path.append("../../rnn")
+from rnn_numpy import GRUCell, LSTMCell, SimpleRNNCell
+
+
+class TestSimpleRNNCell(unittest.TestCase):
+    def __init__(self, bias=True, place="cpu"):
+        super().__init__(methodName="runTest")
+        self.bias = bias
+        self.place = (
+            paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0)
+        )
+
+    def test_with_initial_state(self):
+        rnn1 = SimpleRNNCell(16, 32, bias=self.bias)
+
+        mp = paddle.static.Program()
+        sp = paddle.static.Program()
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            rnn2 = paddle.nn.SimpleRNNCell(
+                16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias
+            )
+
+        place = self.place
+        exe = paddle.static.Executor(place)
+        scope = paddle.base.Scope()
+        with paddle.static.scope_guard(scope):
+            exe.run(sp)
+            convert_params_for_cell_static(rnn1, rnn2, place)
+
+        x = np.random.randn(4, 16)
+        prev_h = np.random.randn(4, 32)
+
+        y1, h1 = rnn1(x, prev_h)
+
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            x_data = paddle.static.data(
+                "input",
+                [-1, 16],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            init_h = paddle.static.data(
+                "init_h",
+                [-1, 32],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            y, h = rnn2(x_data, init_h)
+
+        feed_dict = {x_data.name: x, init_h.name: prev_h}
+        with paddle.static.scope_guard(scope):
+            y2, h2 = exe.run(mp, feed=feed_dict, fetch_list=[y, h])
+
+        np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5)
+
+    def test_with_zero_state(self):
+        rnn1 = SimpleRNNCell(16, 32, bias=self.bias)
+
+        mp = paddle.static.Program()
+        sp = paddle.static.Program()
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            rnn2 = paddle.nn.SimpleRNNCell(
+                16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias
+            )
+
+        place = self.place
+        exe = paddle.static.Executor(place)
+        scope = paddle.base.Scope()
+        with paddle.static.scope_guard(scope):
+            exe.run(sp)
+            convert_params_for_cell_static(rnn1, rnn2, place)
+
+        x = np.random.randn(4, 16)
+
+        y1, h1 = rnn1(x)
+
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            x_data = paddle.static.data(
+                "input",
+                [-1, 16],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            y, h = rnn2(x_data)
+
+        feed_dict = {x_data.name: x}
+
+        with paddle.static.scope_guard(scope):
+            y2, h2 = exe.run(
+                mp, feed=feed_dict, fetch_list=[y, h], use_prune=True
+            )
+
+        np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5)
+
+    def runTest(self):
+        self.test_with_initial_state()
+        self.test_with_zero_state()
+
+
+class TestGRUCell(unittest.TestCase):
+    def __init__(self, bias=True, place="cpu"):
+        super().__init__(methodName="runTest")
+        self.bias = bias
+        self.place = (
+            paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0)
+        )
+
+    def test_with_initial_state(self):
+        rnn1 = GRUCell(16, 32, bias=self.bias)
+
+        mp = paddle.static.Program()
+        sp = paddle.static.Program()
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            rnn2 = paddle.nn.GRUCell(
+                16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias
+            )
+
+        place = self.place
+        exe = paddle.static.Executor(place)
+        scope = paddle.base.Scope()
+        with paddle.static.scope_guard(scope):
+            exe.run(sp)
+            convert_params_for_cell_static(rnn1, rnn2, place)
+
+        x = np.random.randn(4, 16)
+        prev_h = np.random.randn(4, 32)
+
+        y1, h1 = rnn1(x, prev_h)
+
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            x_data = paddle.static.data(
+                "input",
+                [-1, 16],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            init_h = paddle.static.data(
+                "init_h",
+                [-1, 32],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            y, h = rnn2(x_data, init_h)
+
+        feed_dict = {x_data.name: x, init_h.name: prev_h}
+        with paddle.static.scope_guard(scope):
+            y2, h2 = exe.run(mp, feed=feed_dict, fetch_list=[y, h])
+
+        np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5)
+
+    def test_with_zero_state(self):
+        rnn1 = GRUCell(16, 32, bias=self.bias)
+
+        mp = paddle.static.Program()
+        sp = paddle.static.Program()
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            rnn2 = paddle.nn.GRUCell(
+                16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias
+            )
+
+        place = self.place
+        exe = paddle.static.Executor(place)
+        scope = paddle.base.Scope()
+        with paddle.static.scope_guard(scope):
+            exe.run(sp)
+            convert_params_for_cell_static(rnn1, rnn2, place)
+
+        x = np.random.randn(4, 16)
+
+        y1, h1 = rnn1(x)
+
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            x_data = paddle.static.data(
+                "input",
+                [-1, 16],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            y, h = rnn2(x_data)
+
+        feed_dict = {x_data.name: x}
+
+        with paddle.static.scope_guard(scope):
+            y2, h2 = exe.run(
+                mp, feed=feed_dict, fetch_list=[y, h], use_prune=True
+            )
+
+        np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5)
+
+    def runTest(self):
+        self.test_with_initial_state()
+        self.test_with_zero_state()
+
+
+class TestLSTMCell(unittest.TestCase):
+    def __init__(self, bias=True, place="cpu"):
+        super().__init__(methodName="runTest")
+        self.bias = bias
+        self.place = (
+            paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0)
+        )
+
+    def test_with_initial_state(self):
+        rnn1 = LSTMCell(16, 32, bias=self.bias)
+
+        mp = paddle.static.Program()
+        sp = paddle.static.Program()
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            rnn2 = paddle.nn.LSTMCell(
+                16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias
+            )
+
+        place = self.place
+        exe = paddle.static.Executor(place)
+        scope = paddle.base.Scope()
+        with paddle.static.scope_guard(scope):
+            exe.run(sp)
+            convert_params_for_cell_static(rnn1, rnn2, place)
+
+        x = np.random.randn(4, 16)
+        prev_h = np.random.randn(4, 32)
+        prev_c = np.random.randn(4, 32)
+
+        y1, (h1, c1) = rnn1(x, (prev_h, prev_c))
+
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            x_data = paddle.static.data(
+                "input",
+                [-1, 16],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            init_h = paddle.static.data(
+                "init_h",
+                [-1, 32],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            init_c = paddle.static.data(
+                "init_c",
+                [-1, 32],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            y, (h, c) = rnn2(x_data, (init_h, init_c))
+
+        feed_dict = {x_data.name: x, init_h.name: prev_h, init_c.name: prev_c}
+        with paddle.static.scope_guard(scope):
+            y2, h2, c2 = exe.run(mp, feed=feed_dict, fetch_list=[y, h, c])
+
+        np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(c1, c2, atol=1e-8, rtol=1e-5)
+
+    def test_with_zero_state(self):
+        rnn1 = LSTMCell(16, 32, bias=self.bias)
+
+        mp = paddle.static.Program()
+        sp = paddle.static.Program()
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            rnn2 = paddle.nn.LSTMCell(
+                16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias
+            )
+
+        place = self.place
+        exe = paddle.static.Executor(place)
+        scope = paddle.base.Scope()
+        with paddle.static.scope_guard(scope):
+            exe.run(sp)
+            convert_params_for_cell_static(rnn1, rnn2, place)
+
+        x = np.random.randn(4, 16)
+
+        y1, (h1, c1) = rnn1(x)
+
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            x_data = paddle.static.data(
+                "input",
+                [-1, 16],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            y, (h, c) = rnn2(x_data)
+
+        feed_dict = {x_data.name: x}
+
+        with paddle.static.scope_guard(scope):
+            y2, h2, c2 = exe.run(
+                mp, feed=feed_dict, fetch_list=[y, h, c], use_prune=True
+            )
+
+        np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(c1, c2, atol=1e-8, rtol=1e-5)
+
+    def runTest(self):
+        self.test_with_initial_state()
+        self.test_with_zero_state()
+
+
+def load_tests(loader, tests, pattern):
+    suite = unittest.TestSuite()
+    devices = ["cpu", "gpu"] if paddle.base.is_compiled_with_cuda() else ["cpu"]
+    for bias in [True, False]:
+        for device in devices:
+            for test_class in [TestSimpleRNNCell, TestGRUCell, TestLSTMCell]:
+                suite.addTest(test_class(bias, device))
+    return suite
+
+
+if __name__ == "__main__":
+    paddle.enable_static()
+    unittest.main()

test/rnn/test_rnn_cudnn_params_packing.py

@@ -0,0 +1,64 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from unittest import TestCase
+
+import paddle
+
+
+def create_model():
+    hidden_size = 32
+    bilstm = paddle.nn.LSTM(
+        hidden_size, hidden_size, num_layers=1, direction='bidirectional'
+    )
+    return bilstm
+
+
+class TestRNNProgramClone(TestCase):
+    def setUp(self):
+        paddle.enable_static()
+
+    def test_rnn_with_cudnn_clone(self):
+        train_program = paddle.static.Program()
+        test_program = paddle.static.Program()
+        startup_prog = paddle.static.Program()
+
+        # test a typical case in static graph usage: create two nearly
+        # identical program with a shared startup program to share their
+        # parameters
+        #
+        # when creating a parameter, the name is checked. If there is already
+        # a parameter with the same name, which is the output of a operator
+        # (i.e. its creator), its re-creation is skipped.
+        #
+        # but if that parameter has been the output of more than one operator,
+        # an exception is raised. For special cases, white list is added.
+        # flattening rnn's parameters for the need to call cudnn kernel is such
+        # a case.
+        with (
+            paddle.static.program_guard(train_program, startup_prog),
+            paddle.base.unique_name.guard(),
+        ):
+            bilstm = create_model()
+
+        with (
+            paddle.base.program_guard(test_program, startup_prog),
+            paddle.base.unique_name.guard(),
+        ):
+            bilstm = create_model()
+
+
+if __name__ == "__main__":
+    unittest.main()

test/rnn/test_rnn_nets.py

@@ -0,0 +1,356 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle
+
+paddle.set_default_dtype("float64")
+import os
+import sys
+import tempfile
+import unittest
+
+import numpy as np
+from convert import convert_params_for_net
+
+sys.path.append("../../rnn")
+from rnn_numpy import GRU, LSTM, SimpleRNN
+
+bidirectional_list = ["bidirectional", "bidirect"]
+
+
+class TestSimpleRNN(unittest.TestCase):
+    def __init__(
+        self, time_major=True, direction="forward", place="cpu", mode='RNN_TANH'
+    ):
+        super().__init__("runTest")
+        self.time_major = time_major
+        self.direction = direction
+        self.num_directions = 2 if direction in bidirectional_list else 1
+        self.place = place
+        self.mode = mode
+
+    def setUp(self):
+        # Since `set_device` is global, set `set_device` in `setUp` rather than
+        # `__init__` to avoid using an error device set by another test case.
+        place = paddle.set_device(self.place)
+        paddle.disable_static(place)
+        rnn1 = SimpleRNN(
+            16,
+            32,
+            2,
+            time_major=self.time_major,
+            direction=self.direction,
+            nonlinearity=self.mode,
+        )
+        rnn2 = paddle.nn.SimpleRNN(
+            16,
+            32,
+            2,
+            time_major=self.time_major,
+            direction=self.direction,
+            activation=self.mode[4:].lower(),
+        )
+        convert_params_for_net(rnn1, rnn2)
+
+        self.rnn1 = rnn1
+        self.rnn2 = rnn2
+
+    def test_with_initial_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+        prev_h = np.random.randn(2 * self.num_directions, 4, 32)
+
+        y1, h1 = rnn1(x, prev_h)
+        y2, h2 = rnn2(paddle.to_tensor(x), paddle.to_tensor(prev_h))
+        np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_with_zero_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+
+        y1, h1 = rnn1(x)
+        y2, h2 = rnn2(paddle.to_tensor(x))
+        np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_predict(self):
+        predict_test_util(self.place, "SimpleRNN")
+
+    def runTest(self):
+        self.test_with_initial_state()
+        self.test_with_zero_state()
+        self.test_predict()
+
+
+class TestGRU(unittest.TestCase):
+    def __init__(self, time_major=True, direction="forward", place="cpu"):
+        super().__init__("runTest")
+        self.time_major = time_major
+        self.direction = direction
+        self.num_directions = 2 if direction in bidirectional_list else 1
+        self.place = place
+
+    def setUp(self):
+        # Since `set_device` is global, set `set_device` in `setUp` rather than
+        # `__init__` to avoid using an error device set by another test case.
+        place = paddle.set_device(self.place)
+        paddle.disable_static(place)
+        rnn1 = GRU(
+            16, 32, 2, time_major=self.time_major, direction=self.direction
+        )
+        rnn2 = paddle.nn.GRU(
+            16, 32, 2, time_major=self.time_major, direction=self.direction
+        )
+        convert_params_for_net(rnn1, rnn2)
+
+        self.rnn1 = rnn1
+        self.rnn2 = rnn2
+
+    def test_with_initial_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+        prev_h = np.random.randn(2 * self.num_directions, 4, 32)
+
+        y1, h1 = rnn1(x, prev_h)
+        y2, h2 = rnn2(paddle.to_tensor(x), paddle.to_tensor(prev_h))
+        np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_with_zero_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+
+        y1, h1 = rnn1(x)
+        y2, h2 = rnn2(paddle.to_tensor(x))
+        np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_predict(self):
+        predict_test_util(self.place, "GRU")
+
+    def runTest(self):
+        self.test_with_initial_state()
+        self.test_with_zero_state()
+        self.test_predict()
+
+
+class TestLSTM(unittest.TestCase):
+    def __init__(self, time_major=True, direction="forward", place="cpu"):
+        super().__init__("runTest")
+        self.time_major = time_major
+        self.direction = direction
+        self.num_directions = 2 if direction in bidirectional_list else 1
+        self.place = place
+
+    def setUp(self):
+        # Since `set_device` is global, set `set_device` in `setUp` rather than
+        # `__init__` to avoid using an error device set by another test case.
+        place = paddle.set_device(self.place)
+        paddle.disable_static(place)
+        rnn1 = LSTM(
+            16, 32, 2, time_major=self.time_major, direction=self.direction
+        )
+        rnn2 = paddle.nn.LSTM(
+            16, 32, 2, time_major=self.time_major, direction=self.direction
+        )
+        convert_params_for_net(rnn1, rnn2)
+
+        self.rnn1 = rnn1
+        self.rnn2 = rnn2
+
+    def test_with_initial_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+        prev_h = np.random.randn(
+            2 * self.num_directions, 4, getattr(self, "proj_size", 32)
+        )
+        prev_c = np.random.randn(2 * self.num_directions, 4, 32)
+
+        y1, (h1, c1) = rnn1(x, (prev_h, prev_c))
+        y2, (h2, c2) = rnn2(
+            paddle.to_tensor(x),
+            (paddle.to_tensor(prev_h), paddle.to_tensor(prev_c)),
+        )
+        np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(c1, c2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_with_zero_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+
+        y1, (h1, c1) = rnn1(x)
+        y2, (h2, c2) = rnn2(paddle.to_tensor(x))
+        np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(c1, c2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_predict(self):
+        predict_test_util(self.place, "LSTM")
+        predict_test_util(self.place, "LSTM", False)
+
+    def runTest(self):
+        self.test_with_initial_state()
+        self.test_with_zero_state()
+        self.test_predict()
+
+
+class TestLSTMWithProjSize(TestLSTM):
+    def setUp(self):
+        # Since `set_device` is global, set `set_device` in `setUp` rather than
+        # `__init__` to avoid using an error device set by another test case.
+        place = paddle.set_device(self.place)
+        paddle.disable_static(place)
+        rnn1 = LSTM(
+            16,
+            32,
+            2,
+            time_major=self.time_major,
+            direction=self.direction,
+            proj_size=8,
+        )
+        rnn2 = paddle.nn.LSTM(
+            16,
+            32,
+            2,
+            time_major=self.time_major,
+            direction=self.direction,
+            proj_size=8,
+        )
+        convert_params_for_net(rnn1, rnn2)
+
+        self.rnn1 = rnn1
+        self.rnn2 = rnn2
+        self.proj_size = 8
+
+
+def predict_test_util(place, mode, stop_gradient=True):
+    place = paddle.set_device(place)
+    paddle.seed(123)
+    np.random.seed(123)
+
+    class Net(paddle.nn.Layer):
+        def __init__(self):
+            super().__init__()
+            self.rnn = getattr(paddle.nn, mode)(
+                16, 32, 2, direction="bidirectional", dropout=0.1
+            )
+
+        def forward(self, input):
+            return self.rnn(input)
+
+    x = paddle.randn((4, 10, 16))
+    x.stop_gradient = stop_gradient
+    seq_len = paddle.to_tensor(np.array([10, 6, 8, 5]))
+    mask = paddle.static.nn.sequence_lod.sequence_mask(
+        seq_len, maxlen=10, dtype=x.dtype
+    )
+    mask = paddle.unsqueeze(mask, [2])
+    rnn = Net()
+    y, _ = rnn(x)
+    y = y * mask
+    loss = paddle.mean(y)
+    loss.backward()
+    optimizer = paddle.optimizer.Adam(
+        learning_rate=0.1, parameters=rnn.parameters()
+    )
+    optimizer.step()
+    rnn.eval()
+    y, _ = rnn(x)
+    # `jit.to_static` would include a train_program, eval mode might cause
+    # some errors currently, such as dropout grad op gets `is_test == True`.
+    rnn.train()
+
+    rnn = paddle.jit.to_static(
+        rnn,
+        [paddle.static.InputSpec(shape=[None, None, 16], dtype=x.dtype)],
+        full_graph=True,
+    )
+    temp_dir = tempfile.TemporaryDirectory()
+    save_dirname = os.path.join(temp_dir.name, f"./inference/{mode}_infer")
+
+    paddle.jit.save(rnn, save_dirname)
+
+    paddle.enable_static()
+
+    new_scope = paddle.static.Scope()
+    with paddle.static.scope_guard(new_scope):
+        exe = paddle.static.Executor(place)
+        [
+            inference_program,
+            feed_target_names,
+            fetch_targets,
+        ] = paddle.static.load_inference_model(save_dirname, exe)
+        results = exe.run(
+            inference_program,
+            feed={feed_target_names[0]: x.numpy()},
+            fetch_list=fetch_targets,
+        )
+        np.testing.assert_equal(
+            y.numpy(), results[0]
+        )  # eval results equal predict results
+    paddle.disable_static()
+
+    temp_dir.cleanup()
+
+
+def load_tests(loader, tests, pattern):
+    suite = unittest.TestSuite()
+    devices = ["cpu", "gpu"] if paddle.base.is_compiled_with_cuda() else ["cpu"]
+    for direction in ["forward", "bidirectional", "bidirect"]:
+        for time_major in [True, False]:
+            for device in devices:
+                for test_class in [
+                    TestSimpleRNN,
+                    TestLSTM,
+                    TestGRU,
+                    TestLSTMWithProjSize,
+                ]:
+                    suite.addTest(test_class(time_major, direction, device))
+                    if test_class == TestSimpleRNN:
+                        suite.addTest(
+                            test_class(
+                                time_major, direction, device, mode="RNN_RELU"
+                            )
+                        )
+    return suite
+
+
+if __name__ == '__main__':
+    unittest.main()

test/rnn/test_rnn_nets_static.py

@@ -0,0 +1,607 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle
+
+paddle.set_default_dtype("float64")
+
+
+paddle.enable_static()
+
+import sys
+import unittest
+
+import numpy as np
+from convert import convert_params_for_net_static
+
+sys.path.append("../../rnn")
+from rnn_numpy import GRU, LSTM, SimpleRNN
+
+bidirectional_list = ["bidirectional", "bidirect"]
+
+
+class TestSimpleRNN(unittest.TestCase):
+    def __init__(
+        self, time_major=True, direction="forward", place="cpu", mode="RNN_TANH"
+    ):
+        super().__init__("runTest")
+        self.time_major = time_major
+        self.direction = direction
+        self.num_directions = 2 if direction in bidirectional_list else 1
+        self.place = place
+        self.mode = mode
+
+    def test_with_initial_state(self):
+        place = paddle.set_device(self.place)
+        rnn1 = SimpleRNN(
+            16,
+            32,
+            2,
+            time_major=self.time_major,
+            direction=self.direction,
+            nonlinearity=self.mode,
+        )
+
+        mp = paddle.static.Program()
+        sp = paddle.static.Program()
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            rnn2 = paddle.nn.SimpleRNN(
+                16,
+                32,
+                2,
+                time_major=self.time_major,
+                direction=self.direction,
+                activation=self.mode[4:].lower(),
+            )
+
+        exe = paddle.static.Executor(place)
+        scope = paddle.base.Scope()
+        with paddle.static.scope_guard(scope):
+            exe.run(sp)
+            convert_params_for_net_static(rnn1, rnn2, place)
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+        prev_h = np.random.randn(2 * self.num_directions, 4, 32)
+
+        y1, h1 = rnn1(x, prev_h)
+
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            x_data = paddle.static.data(
+                "input",
+                [-1, -1, 16],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            init_h = paddle.static.data(
+                "init_h",
+                [2 * self.num_directions, -1, 32],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            y, h = rnn2(x_data, init_h)
+
+        feed_dict = {x_data.name: x, init_h.name: prev_h}
+        with paddle.static.scope_guard(scope):
+            y2, h2 = exe.run(mp, feed=feed_dict, fetch_list=[y, h])
+
+        np.testing.assert_allclose(y1, y2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5)
+
+    def test_with_zero_state(self):
+        place = paddle.set_device(self.place)
+        rnn1 = SimpleRNN(
+            16,
+            32,
+            2,
+            time_major=self.time_major,
+            direction=self.direction,
+            nonlinearity=self.mode,
+        )
+
+        mp = paddle.static.Program()
+        sp = paddle.static.Program()
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            rnn2 = paddle.nn.SimpleRNN(
+                16,
+                32,
+                2,
+                time_major=self.time_major,
+                direction=self.direction,
+                activation=self.mode[4:].lower(),
+            )
+
+        exe = paddle.static.Executor(place)
+        scope = paddle.base.Scope()
+        with paddle.static.scope_guard(scope):
+            exe.run(sp)
+            convert_params_for_net_static(rnn1, rnn2, place)
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+
+        y1, h1 = rnn1(x)
+
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            x_data = paddle.static.data(
+                "input",
+                [-1, -1, 16],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            y, h = rnn2(x_data)
+
+        feed_dict = {x_data.name: x}
+
+        with paddle.static.scope_guard(scope):
+            y2, h2 = exe.run(mp, feed=feed_dict, fetch_list=[y, h])
+
+        np.testing.assert_allclose(y1, y2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5)
+
+    def runTest(self):
+        self.test_with_initial_state()
+        self.test_with_zero_state()
+
+
+class TestGRU(unittest.TestCase):
+    def __init__(self, time_major=True, direction="forward", place="cpu"):
+        super().__init__("runTest")
+        self.time_major = time_major
+        self.direction = direction
+        self.num_directions = 2 if direction in bidirectional_list else 1
+        self.place = place
+
+    def test_with_initial_state(self):
+        # Since `set_device` is global, set `set_device` in `setUp` rather than
+        # `__init__` to avoid using an error device set by another test case.
+        place = paddle.set_device(self.place)
+        rnn1 = GRU(
+            16, 32, 2, time_major=self.time_major, direction=self.direction
+        )
+
+        mp = paddle.static.Program()
+        sp = paddle.static.Program()
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            rnn2 = paddle.nn.GRU(
+                16,
+                32,
+                2,
+                time_major=self.time_major,
+                direction=self.direction,
+            )
+
+        exe = paddle.static.Executor(place)
+        scope = paddle.base.Scope()
+        with paddle.static.scope_guard(scope):
+            exe.run(sp)
+            convert_params_for_net_static(rnn1, rnn2, place)
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+
+        prev_h = np.random.randn(2 * self.num_directions, 4, 32)
+
+        y1, h1 = rnn1(x, prev_h)
+
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            x_data = paddle.static.data(
+                "input",
+                [-1, -1, 16],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            init_h = paddle.static.data(
+                "init_h",
+                [2 * self.num_directions, -1, 32],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            y, h = rnn2(x_data, init_h)
+
+        feed_dict = {x_data.name: x, init_h.name: prev_h}
+        with paddle.static.scope_guard(scope):
+            y2, h2 = exe.run(mp, feed=feed_dict, fetch_list=[y, h])
+
+        np.testing.assert_allclose(y1, y2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5)
+
+    def test_with_zero_state(self):
+        # Since `set_device` is global, set `set_device` in `setUp` rather than
+        # `__init__` to avoid using an error device set by another test case.
+        place = paddle.set_device(self.place)
+        rnn1 = GRU(
+            16, 32, 2, time_major=self.time_major, direction=self.direction
+        )
+
+        mp = paddle.static.Program()
+        sp = paddle.static.Program()
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            rnn2 = paddle.nn.GRU(
+                16,
+                32,
+                2,
+                time_major=self.time_major,
+                direction=self.direction,
+            )
+
+        exe = paddle.static.Executor(place)
+        scope = paddle.base.Scope()
+        with paddle.static.scope_guard(scope):
+            exe.run(sp)
+            convert_params_for_net_static(rnn1, rnn2, place)
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+
+        y1, h1 = rnn1(x)
+
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            x_data = paddle.static.data(
+                "input",
+                [-1, -1, 16],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            y, h = rnn2(x_data)
+
+        feed_dict = {x_data.name: x}
+
+        with paddle.static.scope_guard(scope):
+            y2, h2 = exe.run(mp, feed=feed_dict, fetch_list=[y, h])
+
+        np.testing.assert_allclose(y1, y2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5)
+
+    def runTest(self):
+        self.test_with_initial_state()
+        self.test_with_zero_state()
+
+
+class TestLSTM(unittest.TestCase):
+    def __init__(self, time_major=True, direction="forward", place="cpu"):
+        super().__init__("runTest")
+        self.time_major = time_major
+        self.direction = direction
+        self.num_directions = 2 if direction in bidirectional_list else 1
+        self.place = place
+
+    def test_with_initial_state(self):
+        # Since `set_device` is global, set `set_device` in `setUp` rather than
+        # `__init__` to avoid using an error device set by another test case.
+        place = paddle.set_device(self.place)
+        rnn1 = LSTM(
+            16, 32, 2, time_major=self.time_major, direction=self.direction
+        )
+
+        mp = paddle.static.Program()
+        sp = paddle.static.Program()
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            rnn2 = paddle.nn.LSTM(
+                16,
+                32,
+                2,
+                time_major=self.time_major,
+                direction=self.direction,
+            )
+
+        exe = paddle.static.Executor(place)
+        scope = paddle.base.Scope()
+        with paddle.static.scope_guard(scope):
+            exe.run(sp)
+            convert_params_for_net_static(rnn1, rnn2, place)
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+        prev_h = np.random.randn(
+            2 * self.num_directions, 4, getattr(self, "proj_size", 32)
+        )
+        prev_c = np.random.randn(2 * self.num_directions, 4, 32)
+
+        y1, (h1, c1) = rnn1(x, (prev_h, prev_c))
+
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            x_data = paddle.static.data(
+                "input",
+                [-1, -1, 16],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            init_h = paddle.static.data(
+                "init_h",
+                [
+                    2 * self.num_directions,
+                    -1,
+                    getattr(self, "proj_size", 32),
+                ],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            init_c = paddle.static.data(
+                "init_c",
+                [2 * self.num_directions, -1, 32],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            y, (h, c) = rnn2(x_data, (init_h, init_c))
+
+        feed_dict = {x_data.name: x, init_h.name: prev_h, init_c.name: prev_c}
+        with paddle.static.scope_guard(scope):
+            y2, h2, c2 = exe.run(mp, feed=feed_dict, fetch_list=[y, h, c])
+
+        np.testing.assert_allclose(y1, y2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(c1, c2, atol=1e-8, rtol=1e-5)
+
+    def test_with_zero_state(self):
+        # Since `set_device` is global, set `set_device` in `setUp` rather than
+        # `__init__` to avoid using an error device set by another test case.
+        place = paddle.set_device(self.place)
+        rnn1 = LSTM(
+            16, 32, 2, time_major=self.time_major, direction=self.direction
+        )
+
+        mp = paddle.static.Program()
+        sp = paddle.static.Program()
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            rnn2 = paddle.nn.LSTM(
+                16,
+                32,
+                2,
+                time_major=self.time_major,
+                direction=self.direction,
+            )
+
+        exe = paddle.static.Executor(place)
+        scope = paddle.base.Scope()
+        with paddle.static.scope_guard(scope):
+            exe.run(sp)
+            convert_params_for_net_static(rnn1, rnn2, place)
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+
+        y1, (h1, c1) = rnn1(x)
+
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            x_data = paddle.static.data(
+                "input",
+                [-1, -1, 16],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            y, (h, c) = rnn2(x_data)
+
+        feed_dict = {x_data.name: x}
+
+        with paddle.static.scope_guard(scope):
+            y2, h2, c2 = exe.run(mp, feed=feed_dict, fetch_list=[y, h, c])
+
+        np.testing.assert_allclose(y1, y2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(c1, c2, atol=1e-8, rtol=1e-5)
+
+    def runTest(self):
+        self.test_with_initial_state()
+        self.test_with_zero_state()
+
+
+class TestLSTMWithProjSize(unittest.TestCase):
+    def __init__(self, time_major=True, direction="forward", place="cpu"):
+        super().__init__("runTest")
+        self.time_major = time_major
+        self.direction = direction
+        self.num_directions = 2 if direction in bidirectional_list else 1
+        self.place = place
+
+    def test_with_initial_state(self):
+        # Since `set_device` is global, set `set_device` in `setUp` rather than
+        # `__init__` to avoid using an error device set by another test case.
+        place = paddle.set_device(self.place)
+        rnn1 = LSTM(
+            16,
+            32,
+            2,
+            time_major=self.time_major,
+            direction=self.direction,
+            proj_size=8,
+        )
+
+        mp = paddle.static.Program()
+        sp = paddle.static.Program()
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            rnn2 = paddle.nn.LSTM(
+                16,
+                32,
+                2,
+                time_major=self.time_major,
+                direction=self.direction,
+                proj_size=8,
+            )
+
+        exe = paddle.static.Executor(place)
+        scope = paddle.base.Scope()
+        with paddle.static.scope_guard(scope):
+            exe.run(sp)
+            convert_params_for_net_static(rnn1, rnn2, place)
+        self.proj_size = 8
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+        prev_h = np.random.randn(
+            2 * self.num_directions, 4, getattr(self, "proj_size", 32)
+        )
+        prev_c = np.random.randn(2 * self.num_directions, 4, 32)
+
+        y1, (h1, c1) = rnn1(x, (prev_h, prev_c))
+
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            x_data = paddle.static.data(
+                "input",
+                [-1, -1, 16],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            init_h = paddle.static.data(
+                "init_h",
+                [
+                    2 * self.num_directions,
+                    -1,
+                    getattr(self, "proj_size", 32),
+                ],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            init_c = paddle.static.data(
+                "init_c",
+                [2 * self.num_directions, -1, 32],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            y, (h, c) = rnn2(x_data, (init_h, init_c))
+
+        feed_dict = {x_data.name: x, init_h.name: prev_h, init_c.name: prev_c}
+        with paddle.static.scope_guard(scope):
+            y2, h2, c2 = exe.run(mp, feed=feed_dict, fetch_list=[y, h, c])
+
+        np.testing.assert_allclose(y1, y2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(c1, c2, atol=1e-8, rtol=1e-5)
+
+    def test_with_zero_state(self):
+        # Since `set_device` is global, set `set_device` in `setUp` rather than
+        # `__init__` to avoid using an error device set by another test case.
+        place = paddle.set_device(self.place)
+        rnn1 = LSTM(
+            16,
+            32,
+            2,
+            time_major=self.time_major,
+            direction=self.direction,
+            proj_size=8,
+        )
+
+        mp = paddle.static.Program()
+        sp = paddle.static.Program()
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            rnn2 = paddle.nn.LSTM(
+                16,
+                32,
+                2,
+                time_major=self.time_major,
+                direction=self.direction,
+                proj_size=8,
+            )
+
+        exe = paddle.static.Executor(place)
+        scope = paddle.base.Scope()
+        with paddle.static.scope_guard(scope):
+            exe.run(sp)
+            convert_params_for_net_static(rnn1, rnn2, place)
+        self.proj_size = 8
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+
+        y1, (h1, c1) = rnn1(x)
+
+        with (
+            paddle.base.unique_name.guard(),
+            paddle.static.program_guard(mp, sp),
+        ):
+            x_data = paddle.static.data(
+                "input",
+                [-1, -1, 16],
+                dtype=paddle.framework.get_default_dtype(),
+            )
+            y, (h, c) = rnn2(x_data)
+
+        feed_dict = {x_data.name: x}
+
+        with paddle.static.scope_guard(scope):
+            y2, h2, c2 = exe.run(mp, feed=feed_dict, fetch_list=[y, h, c])
+
+        np.testing.assert_allclose(y1, y2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(c1, c2, atol=1e-8, rtol=1e-5)
+
+    def runTest(self):
+        self.test_with_initial_state()
+        self.test_with_zero_state()
+
+
+def load_tests(loader, tests, pattern):
+    suite = unittest.TestSuite()
+    devices = ["cpu", "gpu"] if paddle.base.is_compiled_with_cuda() else ["cpu"]
+    for direction in ["forward", "bidirectional", "bidirect"]:
+        for time_major in [True, False]:
+            for device in devices:
+                for test_class in [
+                    TestSimpleRNN,
+                    TestLSTM,
+                    TestGRU,
+                    TestLSTMWithProjSize,
+                ]:
+                    suite.addTest(test_class(time_major, direction, device))
+                    if test_class == TestSimpleRNN:
+                        suite.addTest(
+                            test_class(
+                                time_major, direction, device, mode="RNN_RELU"
+                            )
+                        )
+    return suite
+
+
+if __name__ == "__main__":
+    unittest.main()

test/rnn/test_wrappers.py

@@ -0,0 +1,204 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle
+
+paddle.set_default_dtype("float64")
+import unittest
+
+import numpy as np
+from convert import convert_params_for_cell
+from rnn_numpy import RNN, BiRNN, GRUCell
+
+
+class TestRNNWrapper(unittest.TestCase):
+    def __init__(self, time_major=True, direction="forward", place="cpu"):
+        super().__init__("runTest")
+        self.time_major = time_major
+        self.direction = direction
+        self.place = (
+            paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0)
+        )
+
+    def setUp(self):
+        paddle.disable_static(self.place)
+        cell1 = GRUCell(16, 32)
+        cell2 = paddle.nn.GRUCell(16, 32)
+        convert_params_for_cell(cell1, cell2)
+        rnn1 = RNN(
+            cell1,
+            is_reverse=self.direction == "backward",
+            time_major=self.time_major,
+        )
+        rnn2 = paddle.nn.RNN(
+            cell2,
+            is_reverse=self.direction == "backward",
+            time_major=self.time_major,
+        )
+
+        self.rnn1 = rnn1
+        self.rnn2 = rnn2
+
+    def test_with_initial_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+        prev_h = np.random.randn(4, 32)
+
+        y1, h1 = rnn1(x, prev_h)
+        y2, h2 = rnn2(paddle.to_tensor(x), paddle.to_tensor(prev_h))
+        np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_with_zero_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+
+        y1, h1 = rnn1(x)
+        y2, h2 = rnn2(paddle.to_tensor(x))
+        np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_with_input_lengths(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+        sequence_length = np.array([12, 10, 9, 8], dtype=np.int64)
+
+        y1, h1 = rnn1(x, sequence_length=sequence_length)
+
+        seq_len = paddle.to_tensor(sequence_length)
+        mask = paddle.static.nn.sequence_lod.sequence_mask(
+            seq_len, dtype=paddle.get_default_dtype()
+        )
+        if self.time_major:
+            mask = paddle.transpose(mask, [1, 0])
+        y2, h2 = rnn2(paddle.to_tensor(x), sequence_length=seq_len)
+        mask = paddle.unsqueeze(mask, -1)
+        y2 = paddle.multiply(y2, mask)
+
+        np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def runTest(self):
+        self.test_with_initial_state()
+        self.test_with_zero_state()
+        self.test_with_input_lengths()
+
+
+class TestBiRNNWrapper(unittest.TestCase):
+    def __init__(self, time_major=True, place="cpu"):
+        super().__init__("runTest")
+        self.time_major = time_major
+        self.place = (
+            paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0)
+        )
+
+    def setUp(self):
+        paddle.disable_static(self.place)
+        fw_cell1 = GRUCell(16, 32)
+        bw_cell1 = GRUCell(16, 32)
+        fw_cell2 = paddle.nn.GRUCell(16, 32)
+        bw_cell2 = paddle.nn.GRUCell(16, 32)
+        convert_params_for_cell(fw_cell1, fw_cell2)
+        convert_params_for_cell(bw_cell1, bw_cell2)
+        rnn1 = BiRNN(fw_cell1, bw_cell1, time_major=self.time_major)
+        rnn2 = paddle.nn.BiRNN(fw_cell2, bw_cell2, time_major=self.time_major)
+
+        self.rnn1 = rnn1
+        self.rnn2 = rnn2
+
+    def test_with_initial_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+        fw_prev_h = np.random.randn(4, 32)
+        bw_prev_h = np.random.randn(4, 32)
+
+        y1, (fw_h1, bw_h1) = rnn1(x, (fw_prev_h, bw_prev_h))
+        y2, (fw_h2, bw_h2) = rnn2(
+            paddle.to_tensor(x),
+            (paddle.to_tensor(fw_prev_h), paddle.to_tensor(bw_prev_h)),
+        )
+        np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(fw_h1, fw_h2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(bw_h1, bw_h2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_with_zero_state(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+
+        y1, (fw_h1, bw_h1) = rnn1(x)
+        y2, (fw_h2, bw_h2) = rnn2(paddle.to_tensor(x))
+        np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(fw_h1, fw_h2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(bw_h1, bw_h2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def test_with_input_lengths(self):
+        rnn1 = self.rnn1
+        rnn2 = self.rnn2
+
+        x = np.random.randn(12, 4, 16)
+        if not self.time_major:
+            x = np.transpose(x, [1, 0, 2])
+        sequence_length = np.array([12, 10, 9, 8], dtype=np.int64)
+
+        y1, (fw_h1, bw_h1) = rnn1(x, sequence_length=sequence_length)
+
+        seq_len = paddle.to_tensor(sequence_length)
+        mask = paddle.static.nn.sequence_lod.sequence_mask(
+            seq_len, dtype=paddle.get_default_dtype()
+        )
+        if self.time_major:
+            mask = paddle.transpose(mask, [1, 0])
+        y2, (fw_h2, bw_h2) = rnn2(paddle.to_tensor(x), sequence_length=seq_len)
+        mask = paddle.unsqueeze(mask, -1)
+        y2 = paddle.multiply(y2, mask)
+
+        np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(fw_h1, fw_h2.numpy(), atol=1e-8, rtol=1e-5)
+        np.testing.assert_allclose(bw_h1, bw_h2.numpy(), atol=1e-8, rtol=1e-5)
+
+    def runTest(self):
+        self.test_with_initial_state()
+        self.test_with_zero_state()
+        self.test_with_input_lengths()
+
+
+def load_tests(loader, tests, pattern):
+    suite = unittest.TestSuite()
+    devices = ["cpu", "gpu"] if paddle.base.is_compiled_with_cuda() else ["cpu"]
+    for direction in ["forward", "backward"]:
+        for device in devices:
+            for time_major in [False]:
+                suite.addTest(TestRNNWrapper(time_major, direction, device))
+            suite.addTest(TestBiRNNWrapper(time_major, device))
+    return suite