chore: import upstream snapshot with attribution

mla/neuralnet/__init__.py

@@ -0,0 +1 @@
+from .nnet import NeuralNet

mla/neuralnet/activations.py

@@ -0,0 +1,64 @@
+import autograd.numpy as np
+
+"""
+References:
+https://en.wikipedia.org/wiki/Activation_function
+"""
+
+
+def sigmoid(z):
+    return 1.0 / (1.0 + np.exp(-z))
+
+
+def softmax(z):
+    # Avoid numerical overflow by removing max
+    e = np.exp(z - np.amax(z, axis=1, keepdims=True))
+    return e / np.sum(e, axis=1, keepdims=True)
+
+
+def linear(z):
+    return z
+
+
+def softplus(z):
+    """Smooth relu."""
+    # Avoid numerical overflow, see:
+    # https://docs.scipy.org/doc/numpy/reference/generated/numpy.logaddexp.html
+    return np.logaddexp(0.0, z)
+
+
+def softsign(z):
+    return z / (1 + np.abs(z))
+
+
+def tanh(z):
+    return np.tanh(z)
+
+
+def relu(z):
+    return np.maximum(0, z)
+
+
+def leakyrelu(z, a=0.01):
+    return np.maximum(z * a, z)
+
+
+def gelu(z):
+    """
+    Gaussian Error Linear Unit (GELU)
+    """
+    # mainly used in transformers smoother version of relu
+    
+    return 0.5 * z * (
+        1.0 + np.tanh(
+            np.sqrt(2.0 / np.pi) * (z + 0.044715 * np.power(z, 3))
+        )
+    )
+
+
+def get_activation(name):
+    """Return activation function by name"""
+    try:
+        return globals()[name]
+    except Exception:
+        raise ValueError("Invalid activation function.")

mla/neuralnet/constraints.py

@@ -0,0 +1,40 @@
+# coding:utf-8
+import numpy as np
+
+EPSILON = 10e-8
+
+
+class Constraint(object):
+    def clip(self, p):
+        return p
+
+
+class MaxNorm(object):
+    def __init__(self, m=2, axis=0):
+        self.axis = axis
+        self.m = m
+
+    def clip(self, p):
+        norms = np.sqrt(np.sum(p**2, axis=self.axis))
+        desired = np.clip(norms, 0, self.m)
+        p = p * (desired / (EPSILON + norms))
+        return p
+
+
+class NonNeg(object):
+    def clip(self, p):
+        p[p < 0.0] = 0.0
+        return p
+
+
+class SmallNorm(object):
+    def clip(self, p):
+        return np.clip(p, -5, 5)
+
+
+class UnitNorm(Constraint):
+    def __init__(self, axis=0):
+        self.axis = axis
+
+    def clip(self, p):
+        return p / (EPSILON + np.sqrt(np.sum(p**2, axis=self.axis)))

mla/neuralnet/initializations.py

@@ -0,0 +1,75 @@
+import numpy as np
+
+"""
+References:
+http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf
+
+"""
+
+
+def normal(shape, scale=0.5):
+    return np.random.normal(size=shape, scale=scale)
+
+
+def uniform(shape, scale=0.5):
+    return np.random.uniform(size=shape, low=-scale, high=scale)
+
+
+def zero(shape, **kwargs):
+    return np.zeros(shape)
+
+
+def one(shape, **kwargs):
+    return np.ones(shape)
+
+
+def orthogonal(shape, scale=0.5):
+    flat_shape = (shape[0], np.prod(shape[1:]))
+    array = np.random.normal(size=flat_shape)
+    u, _, v = np.linalg.svd(array, full_matrices=False)
+    array = u if u.shape == flat_shape else v
+    return np.reshape(array * scale, shape)
+
+
+def _glorot_fan(shape):
+    assert len(shape) >= 2
+
+    if len(shape) == 4:
+        receptive_field_size = np.prod(shape[2:])
+        fan_in = shape[1] * receptive_field_size
+        fan_out = shape[0] * receptive_field_size
+    else:
+        fan_in, fan_out = shape[:2]
+    return float(fan_in), float(fan_out)
+
+
+def glorot_normal(shape, **kwargs):
+    fan_in, fan_out = _glorot_fan(shape)
+    s = np.sqrt(2.0 / (fan_in + fan_out))
+    return normal(shape, s)
+
+
+def glorot_uniform(shape, **kwargs):
+    fan_in, fan_out = _glorot_fan(shape)
+    s = np.sqrt(6.0 / (fan_in + fan_out))
+    return uniform(shape, s)
+
+
+def he_normal(shape, **kwargs):
+    fan_in, fan_out = _glorot_fan(shape)
+    s = np.sqrt(2.0 / fan_in)
+    return normal(shape, s)
+
+
+def he_uniform(shape, **kwargs):
+    fan_in, fan_out = _glorot_fan(shape)
+    s = np.sqrt(6.0 / fan_in)
+    return uniform(shape, s)
+
+
+def get_initializer(name):
+    """Returns initialization function by the name."""
+    try:
+        return globals()[name]
+    except Exception:
+        raise ValueError("Invalid initialization function.")

mla/neuralnet/layers/__init__.py

@@ -0,0 +1,4 @@
+# coding:utf-8
+from .basic import *
+from .convnet import *
+from .normalization import *

mla/neuralnet/layers/basic.py

@@ -0,0 +1,183 @@
+# coding:utf-8
+import autograd.numpy as np
+from autograd import elementwise_grad
+
+from mla.neuralnet.activations import get_activation
+from mla.neuralnet.parameters import Parameters
+
+np.random.seed(9999)
+
+
+class Layer(object):
+    def setup(self, X_shape):
+        """Allocates initial weights."""
+        pass
+
+    def forward_pass(self, x):
+        raise NotImplementedError()
+
+    def backward_pass(self, delta):
+        raise NotImplementedError()
+
+    def shape(self, x_shape):
+        """Returns shape of the current layer."""
+        raise NotImplementedError()
+
+
+class ParamMixin(object):
+    @property
+    def parameters(self):
+        return self._params
+
+
+class PhaseMixin(object):
+    _train = False
+
+    @property
+    def is_training(self):
+        return self._train
+
+    @is_training.setter
+    def is_training(self, is_train=True):
+        self._train = is_train
+
+    @property
+    def is_testing(self):
+        return not self._train
+
+    @is_testing.setter
+    def is_testing(self, is_test=True):
+        self._train = not is_test
+
+
+class Dense(Layer, ParamMixin):
+    def __init__(self, output_dim, parameters=None):
+        """A fully connected layer.
+
+        Parameters
+        ----------
+        output_dim : int
+        """
+        self._params = parameters
+        self.output_dim = output_dim
+        self.last_input = None
+
+        if parameters is None:
+            self._params = Parameters()
+
+    def setup(self, x_shape):
+        self._params.setup_weights((x_shape[1], self.output_dim))
+
+    def forward_pass(self, X):
+        self.last_input = X
+        return self.weight(X)
+
+    def weight(self, X):
+        W = np.dot(X, self._params["W"])
+        return W + self._params["b"]
+
+    def backward_pass(self, delta):
+        dW = np.dot(self.last_input.T, delta)
+        db = np.sum(delta, axis=0)
+
+        # Update gradient values
+        self._params.update_grad("W", dW)
+        self._params.update_grad("b", db)
+        return np.dot(delta, self._params["W"].T)
+
+    def shape(self, x_shape):
+        return x_shape[0], self.output_dim
+
+
+class Activation(Layer):
+    def __init__(self, name):
+        self.last_input = None
+        self.activation = get_activation(name)
+        # Derivative of activation function
+        self.activation_d = elementwise_grad(self.activation)
+
+    def forward_pass(self, X):
+        self.last_input = X
+        return self.activation(X)
+
+    def backward_pass(self, delta):
+        return self.activation_d(self.last_input) * delta
+
+    def shape(self, x_shape):
+        return x_shape
+
+
+class Dropout(Layer, PhaseMixin):
+    """Randomly set a fraction of `p` inputs to 0 at each training update."""
+
+    def __init__(self, p=0.1):
+        self.p = p
+        self._mask = None
+
+    def forward_pass(self, X):
+        assert self.p > 0
+        if self.is_training:
+            self._mask = np.random.uniform(size=X.shape) > self.p
+            y = X * self._mask
+        else:
+            y = X * (1.0 - self.p)
+
+        return y
+
+    def backward_pass(self, delta):
+        return delta * self._mask
+
+    def shape(self, x_shape):
+        return x_shape
+
+
+class TimeStepSlicer(Layer):
+    """Take a specific time step from 3D tensor."""
+
+    def __init__(self, step=-1):
+        self.step = step
+
+    def forward_pass(self, x):
+        return x[:, self.step, :]
+
+    def backward_pass(self, delta):
+        return np.repeat(delta[:, np.newaxis, :], 2, 1)
+
+    def shape(self, x_shape):
+        return x_shape[0], x_shape[2]
+
+
+class TimeDistributedDense(Layer):
+    """Apply regular Dense layer to every timestep."""
+
+    def __init__(self, output_dim):
+        self.output_dim = output_dim
+        self.n_timesteps = None
+        self.dense = None
+        self.input_dim = None
+
+    def setup(self, X_shape):
+        self.dense = Dense(self.output_dim)
+        self.dense.setup((X_shape[0], X_shape[2]))
+        self.input_dim = X_shape[2]
+
+    def forward_pass(self, X):
+        n_timesteps = X.shape[1]
+        X = X.reshape(-1, X.shape[-1])
+        y = self.dense.forward_pass(X)
+        y = y.reshape((-1, n_timesteps, self.output_dim))
+        return y
+
+    def backward_pass(self, delta):
+        n_timesteps = delta.shape[1]
+        X = delta.reshape(-1, delta.shape[-1])
+        y = self.dense.backward_pass(X)
+        y = y.reshape((-1, n_timesteps, self.input_dim))
+        return y
+
+    @property
+    def parameters(self):
+        return self.dense._params
+
+    def shape(self, x_shape):
+        return x_shape[0], x_shape[1], self.output_dim

mla/neuralnet/layers/convnet.py

@@ -0,0 +1,231 @@
+# coding:utf-8
+import autograd.numpy as np
+
+from mla.neuralnet.layers import Layer, ParamMixin
+from mla.neuralnet.parameters import Parameters
+
+
+class Convolution(Layer, ParamMixin):
+    def __init__(
+        self,
+        n_filters=8,
+        filter_shape=(3, 3),
+        padding=(0, 0),
+        stride=(1, 1),
+        parameters=None,
+    ):
+        """A 2D convolutional layer.
+        Input shape: (n_images, n_channels, height, width)
+
+        Parameters
+        ----------
+        n_filters : int, default 8
+            The number of filters (kernels).
+        filter_shape : tuple(int, int), default (3, 3)
+            The shape of the filters. (height, width)
+        parameters : Parameters instance, default None
+        stride : tuple(int, int), default (1, 1)
+            The step of the convolution. (height, width).
+        padding : tuple(int, int), default (0, 0)
+            The number of pixel to add to each side of the input. (height, weight)
+
+        """
+        self.padding = padding
+        self._params = parameters
+        self.stride = stride
+        self.filter_shape = filter_shape
+        self.n_filters = n_filters
+        if self._params is None:
+            self._params = Parameters()
+
+    def setup(self, X_shape):
+        n_channels, self.height, self.width = X_shape[1:]
+
+        W_shape = (self.n_filters, n_channels) + self.filter_shape
+        b_shape = self.n_filters
+        self._params.setup_weights(W_shape, b_shape)
+
+    def forward_pass(self, X):
+        n_images, n_channels, height, width = self.shape(X.shape)
+        self.last_input = X
+        self.col = image_to_column(X, self.filter_shape, self.stride, self.padding)
+        self.col_W = self._params["W"].reshape(self.n_filters, -1).T
+
+        out = np.dot(self.col, self.col_W) + self._params["b"]
+        out = out.reshape(n_images, height, width, -1).transpose(0, 3, 1, 2)
+        return out
+
+    def backward_pass(self, delta):
+        delta = delta.transpose(0, 2, 3, 1).reshape(-1, self.n_filters)
+
+        d_W = np.dot(self.col.T, delta).transpose(1, 0).reshape(self._params["W"].shape)
+        d_b = np.sum(delta, axis=0)
+        self._params.update_grad("b", d_b)
+        self._params.update_grad("W", d_W)
+
+        d_c = np.dot(delta, self.col_W.T)
+        return column_to_image(
+            d_c, self.last_input.shape, self.filter_shape, self.stride, self.padding
+        )
+
+    def shape(self, x_shape):
+        height, width = convoltuion_shape(
+            self.height, self.width, self.filter_shape, self.stride, self.padding
+        )
+        return x_shape[0], self.n_filters, height, width
+
+
+class MaxPooling(Layer):
+    def __init__(self, pool_shape=(2, 2), stride=(1, 1), padding=(0, 0)):
+        """Max pooling layer.
+        Input shape: (n_images, n_channels, height, width)
+
+        Parameters
+        ----------
+        pool_shape : tuple(int, int), default (2, 2)
+        stride : tuple(int, int), default (1,1)
+        padding : tuple(int, int), default (0,0)
+        """
+        self.pool_shape = pool_shape
+        self.stride = stride
+        self.padding = padding
+
+    def forward_pass(self, X):
+        self.last_input = X
+
+        out_height, out_width = pooling_shape(self.pool_shape, X.shape, self.stride)
+        n_images, n_channels, _, _ = X.shape
+
+        col = image_to_column(X, self.pool_shape, self.stride, self.padding)
+        col = col.reshape(-1, self.pool_shape[0] * self.pool_shape[1])
+
+        arg_max = np.argmax(col, axis=1)
+        out = np.max(col, axis=1)
+        self.arg_max = arg_max
+        return out.reshape(n_images, out_height, out_width, n_channels).transpose(
+            0, 3, 1, 2
+        )
+
+    def backward_pass(self, delta):
+        delta = delta.transpose(0, 2, 3, 1)
+
+        pool_size = self.pool_shape[0] * self.pool_shape[1]
+        y_max = np.zeros((delta.size, pool_size))
+        y_max[np.arange(self.arg_max.size), self.arg_max.flatten()] = delta.flatten()
+        y_max = y_max.reshape(delta.shape + (pool_size,))
+
+        dcol = y_max.reshape(y_max.shape[0] * y_max.shape[1] * y_max.shape[2], -1)
+        return column_to_image(
+            dcol, self.last_input.shape, self.pool_shape, self.stride, self.padding
+        )
+
+    def shape(self, x_shape):
+        h, w = convoltuion_shape(
+            x_shape[2], x_shape[3], self.pool_shape, self.stride, self.padding
+        )
+        return x_shape[0], x_shape[1], h, w
+
+
+class Flatten(Layer):
+    """Flattens multidimensional input into 2D matrix."""
+
+    def forward_pass(self, X):
+        self.last_input_shape = X.shape
+        return X.reshape((X.shape[0], -1))
+
+    def backward_pass(self, delta):
+        return delta.reshape(self.last_input_shape)
+
+    def shape(self, x_shape):
+        return x_shape[0], np.prod(x_shape[1:])
+
+
+def image_to_column(images, filter_shape, stride, padding):
+    """Rearrange image blocks into columns.
+
+    Parameters
+    ----------
+
+    filter_shape : tuple(height, width)
+    images : np.array, shape (n_images, n_channels, height, width)
+    padding: tuple(height, width)
+    stride : tuple (height, width)
+
+    """
+    n_images, n_channels, height, width = images.shape
+    f_height, f_width = filter_shape
+    out_height, out_width = convoltuion_shape(
+        height, width, (f_height, f_width), stride, padding
+    )
+    images = np.pad(images, ((0, 0), (0, 0), padding, padding), mode="constant")
+
+    col = np.zeros((n_images, n_channels, f_height, f_width, out_height, out_width))
+    for y in range(f_height):
+        y_bound = y + stride[0] * out_height
+        for x in range(f_width):
+            x_bound = x + stride[1] * out_width
+            col[:, :, y, x, :, :] = images[
+                :, :, y : y_bound : stride[0], x : x_bound : stride[1]
+            ]
+
+    col = col.transpose(0, 4, 5, 1, 2, 3).reshape(n_images * out_height * out_width, -1)
+    return col
+
+
+def column_to_image(columns, images_shape, filter_shape, stride, padding):
+    """Rearrange columns into image blocks.
+
+    Parameters
+    ----------
+    columns
+    images_shape : tuple(n_images, n_channels, height, width)
+    filter_shape : tuple(height, _width)
+    stride : tuple(height, width)
+    padding : tuple(height, width)
+    """
+    n_images, n_channels, height, width = images_shape
+    f_height, f_width = filter_shape
+
+    out_height, out_width = convoltuion_shape(
+        height, width, (f_height, f_width), stride, padding
+    )
+    columns = columns.reshape(
+        n_images, out_height, out_width, n_channels, f_height, f_width
+    ).transpose(0, 3, 4, 5, 1, 2)
+
+    img_h = height + 2 * padding[0] + stride[0] - 1
+    img_w = width + 2 * padding[1] + stride[1] - 1
+    img = np.zeros((n_images, n_channels, img_h, img_w))
+    for y in range(f_height):
+        y_bound = y + stride[0] * out_height
+        for x in range(f_width):
+            x_bound = x + stride[1] * out_width
+            img[:, :, y : y_bound : stride[0], x : x_bound : stride[1]] += columns[
+                :, :, y, x, :, :
+            ]
+
+    return img[:, :, padding[0] : height + padding[0], padding[1] : width + padding[1]]
+
+
+def convoltuion_shape(img_height, img_width, filter_shape, stride, padding):
+    """Calculate output shape for convolution layer."""
+    height = (img_height + 2 * padding[0] - filter_shape[0]) / float(stride[0]) + 1
+    width = (img_width + 2 * padding[1] - filter_shape[1]) / float(stride[1]) + 1
+
+    assert height % 1 == 0
+    assert width % 1 == 0
+
+    return int(height), int(width)
+
+
+def pooling_shape(pool_shape, image_shape, stride):
+    """Calculate output shape for pooling layer."""
+    n_images, n_channels, height, width = image_shape
+
+    height = (height - pool_shape[0]) / float(stride[0]) + 1
+    width = (width - pool_shape[1]) / float(stride[1]) + 1
+
+    assert height % 1 == 0
+    assert width % 1 == 0
+
+    return int(height), int(width)

mla/neuralnet/layers/normalization.py

@@ -0,0 +1,158 @@
+# coding:utf-8
+import numpy as np
+
+from mla.neuralnet.layers import Layer, PhaseMixin, ParamMixin
+from mla.neuralnet.parameters import Parameters
+
+"""
+References:
+https://kratzert.github.io/2016/02/12/understanding-the-gradient-flow-through-the-batch-normalization-layer.html
+"""
+
+
+class BatchNormalization(Layer, ParamMixin, PhaseMixin):
+    def __init__(self, momentum=0.9, eps=1e-5, parameters=None):
+        super().__init__()
+        self._params = parameters
+        if self._params is None:
+            self._params = Parameters()
+        self.momentum = momentum
+        self.eps = eps
+        self.ema_mean = None
+        self.ema_var = None
+
+    def setup(self, x_shape):
+        self._params.setup_weights((1, x_shape[1]))
+
+    def _forward_pass(self, X):
+        gamma = self._params["W"]
+        beta = self._params["b"]
+
+        if self.is_testing:
+            mu = self.ema_mean
+            xmu = X - mu
+            var = self.ema_var
+            sqrtvar = np.sqrt(var + self.eps)
+            ivar = 1.0 / sqrtvar
+            xhat = xmu * ivar
+            gammax = gamma * xhat
+            return gammax + beta
+
+        N, D = X.shape
+
+        # step1: calculate mean
+        mu = 1.0 / N * np.sum(X, axis=0)
+
+        # step2: subtract mean vector of every trainings example
+        xmu = X - mu
+
+        # step3: following the lower branch - calculation denominator
+        sq = xmu**2
+
+        # step4: calculate variance
+        var = 1.0 / N * np.sum(sq, axis=0)
+
+        # step5: add eps for numerical stability, then sqrt
+        sqrtvar = np.sqrt(var + self.eps)
+
+        # step6: invert sqrtwar
+        ivar = 1.0 / sqrtvar
+
+        # step7: execute normalization
+        xhat = xmu * ivar
+
+        # step8: Nor the two transformation steps
+        gammax = gamma * xhat
+
+        # step9
+        out = gammax + beta
+
+        # store running averages of mean and variance during training for use during testing
+        if self.ema_mean is None or self.ema_var is None:
+            self.ema_mean = mu
+            self.ema_var = var
+        else:
+            self.ema_mean = self.momentum * self.ema_mean + (1 - self.momentum) * mu
+            self.ema_var = self.momentum * self.ema_var + (1 - self.momentum) * var
+        # store intermediate
+        self.cache = (xhat, gamma, xmu, ivar, sqrtvar, var)
+
+        return out
+
+    def forward_pass(self, X):
+        if len(X.shape) == 2:
+            # input is a regular layer
+            return self._forward_pass(X)
+        elif len(X.shape) == 4:
+            # input is a convolution layer
+            N, C, H, W = X.shape
+            x_flat = X.transpose(0, 2, 3, 1).reshape(-1, C)
+            out_flat = self._forward_pass(x_flat)
+            return out_flat.reshape(N, H, W, C).transpose(0, 3, 1, 2)
+        else:
+            raise NotImplementedError(
+                "Unknown model with dimensions = {}".format(len(X.shape))
+            )
+
+    def _backward_pass(self, delta):
+        # unfold the variables stored in cache
+        xhat, gamma, xmu, ivar, sqrtvar, var = self.cache
+
+        # get the dimensions of the input/output
+        N, D = delta.shape
+
+        # step9
+        dbeta = np.sum(delta, axis=0)
+        dgammax = delta  # not necessary, but more understandable
+
+        # step8
+        dgamma = np.sum(dgammax * xhat, axis=0)
+        dxhat = dgammax * gamma
+
+        # step7
+        divar = np.sum(dxhat * xmu, axis=0)
+        dxmu1 = dxhat * ivar
+
+        # step6
+        dsqrtvar = -1.0 / (sqrtvar**2) * divar
+
+        # step5
+        dvar = 0.5 * 1.0 / np.sqrt(var + self.eps) * dsqrtvar
+
+        # step4
+        dsq = 1.0 / N * np.ones((N, D)) * dvar
+
+        # step3
+        dxmu2 = 2 * xmu * dsq
+
+        # step2
+        dx1 = dxmu1 + dxmu2
+        dmu = -1 * np.sum(dxmu1 + dxmu2, axis=0)
+
+        # step1
+        dx2 = 1.0 / N * np.ones((N, D)) * dmu
+
+        # step0
+        dx = dx1 + dx2
+
+        # Update gradient values
+        self._params.update_grad("W", dgamma)
+        self._params.update_grad("b", dbeta)
+
+        return dx
+
+    def backward_pass(self, X):
+        if len(X.shape) == 2:
+            # input is a regular layer
+            return self._backward_pass(X)
+        elif len(X.shape) == 4:
+            # input is a convolution layer
+            N, C, H, W = X.shape
+            x_flat = X.transpose(0, 2, 3, 1).reshape(-1, C)
+            out_flat = self._backward_pass(x_flat)
+            return out_flat.reshape(N, H, W, C).transpose(0, 3, 1, 2)
+        else:
+            raise NotImplementedError("Unknown model shape: {}".format(X.shape))
+
+    def shape(self, x_shape):
+        return x_shape

mla/neuralnet/layers/recurrent/__init__.py

@@ -0,0 +1,3 @@
+# coding:utf-8
+from .lstm import *
+from .rnn import *

mla/neuralnet/layers/recurrent/lstm.py

@@ -0,0 +1,195 @@
+# coding:utf-8
+import autograd.numpy as np
+from autograd import elementwise_grad
+
+from mla.neuralnet.activations import sigmoid
+from mla.neuralnet.initializations import get_initializer
+from mla.neuralnet.layers import Layer, get_activation, ParamMixin
+from mla.neuralnet.parameters import Parameters
+
+"""
+References:
+Understanding LSTM Networks http://colah.github.io/posts/2015-08-Understanding-LSTMs/
+A Critical Review of Recurrent Neural Networks for Sequence Learning http://arxiv.org/pdf/1506.00019v4.pdf
+"""
+
+
+class LSTM(Layer, ParamMixin):
+    def __init__(
+        self,
+        hidden_dim,
+        activation="tanh",
+        inner_init="orthogonal",
+        parameters=None,
+        return_sequences=True,
+    ):
+        self.return_sequences = return_sequences
+        self.hidden_dim = hidden_dim
+        self.inner_init = get_initializer(inner_init)
+        self.activation = get_activation(activation)
+        self.activation_d = elementwise_grad(self.activation)
+        self.sigmoid_d = elementwise_grad(sigmoid)
+
+        if parameters is None:
+            self._params = Parameters()
+        else:
+            self._params = parameters
+
+        self.last_input = None
+        self.states = None
+        self.outputs = None
+        self.gates = None
+        self.hprev = None
+        self.input_dim = None
+        self.W = None
+        self.U = None
+
+    def setup(self, x_shape):
+        """
+        Naming convention:
+        i : input gate
+        f : forget gate
+        c : cell
+        o : output gate
+
+        Parameters
+        ----------
+        x_shape : np.array(batch size, time steps, input shape)
+        """
+        self.input_dim = x_shape[2]
+        # Input -> Hidden
+        W_params = ["W_i", "W_f", "W_o", "W_c"]
+        # Hidden -> Hidden
+        U_params = ["U_i", "U_f", "U_o", "U_c"]
+        # Bias terms
+        b_params = ["b_i", "b_f", "b_o", "b_c"]
+
+        # Initialize params
+        for param in W_params:
+            self._params[param] = self._params.init((self.input_dim, self.hidden_dim))
+
+        for param in U_params:
+            self._params[param] = self.inner_init((self.hidden_dim, self.hidden_dim))
+
+        for param in b_params:
+            self._params[param] = np.full((self.hidden_dim,), self._params.initial_bias)
+
+        # Combine weights for simplicity
+        self.W = [self._params[param] for param in W_params]
+        self.U = [self._params[param] for param in U_params]
+
+        # Init gradient arrays for all weights
+        self._params.init_grad()
+
+        self.hprev = np.zeros((x_shape[0], self.hidden_dim))
+        self.oprev = np.zeros((x_shape[0], self.hidden_dim))
+
+    def forward_pass(self, X):
+        n_samples, n_timesteps, input_shape = X.shape
+        p = self._params
+        self.last_input = X
+
+        self.states = np.zeros((n_samples, n_timesteps + 1, self.hidden_dim))
+        self.outputs = np.zeros((n_samples, n_timesteps + 1, self.hidden_dim))
+        self.gates = {
+            k: np.zeros((n_samples, n_timesteps, self.hidden_dim))
+            for k in ["i", "f", "o", "c"]
+        }
+
+        self.states[:, -1, :] = self.hprev
+        self.outputs[:, -1, :] = self.oprev
+
+        for i in range(n_timesteps):
+            t_gates = np.dot(X[:, i, :], self.W) + np.dot(
+                self.outputs[:, i - 1, :], self.U
+            )
+
+            # Input
+            self.gates["i"][:, i, :] = sigmoid(t_gates[:, 0, :] + p["b_i"])
+            # Forget
+            self.gates["f"][:, i, :] = sigmoid(t_gates[:, 1, :] + p["b_f"])
+            # Output
+            self.gates["o"][:, i, :] = sigmoid(t_gates[:, 2, :] + p["b_o"])
+            # Cell
+            self.gates["c"][:, i, :] = self.activation(t_gates[:, 3, :] + p["b_c"])
+
+            # (previous state * forget) + input + cell
+            self.states[:, i, :] = (
+                self.states[:, i - 1, :] * self.gates["f"][:, i, :]
+                + self.gates["i"][:, i, :] * self.gates["c"][:, i, :]
+            )
+            self.outputs[:, i, :] = self.gates["o"][:, i, :] * self.activation(
+                self.states[:, i, :]
+            )
+
+        self.hprev = self.states[:, n_timesteps - 1, :].copy()
+        self.oprev = self.outputs[:, n_timesteps - 1, :].copy()
+
+        if self.return_sequences:
+            return self.outputs[:, 0:-1, :]
+        else:
+            return self.outputs[:, -2, :]
+
+    def backward_pass(self, delta):
+        if len(delta.shape) == 2:
+            delta = delta[:, np.newaxis, :]
+
+        n_samples, n_timesteps, input_shape = delta.shape
+
+        # Temporal gradient arrays
+        grad = {k: np.zeros_like(self._params[k]) for k in self._params.keys()}
+
+        dh_next = np.zeros((n_samples, input_shape))
+        output = np.zeros((n_samples, n_timesteps, self.input_dim))
+
+        # Backpropagation through time
+        for i in reversed(range(n_timesteps)):
+            dhi = (
+                delta[:, i, :]
+                * self.gates["o"][:, i, :]
+                * self.activation_d(self.states[:, i, :])
+                + dh_next
+            )
+
+            og = delta[:, i, :] * self.activation(self.states[:, i, :])
+            de_o = og * self.sigmoid_d(self.gates["o"][:, i, :])
+
+            grad["W_o"] += np.dot(self.last_input[:, i, :].T, de_o)
+            grad["U_o"] += np.dot(self.outputs[:, i - 1, :].T, de_o)
+            grad["b_o"] += de_o.sum(axis=0)
+
+            de_f = (dhi * self.states[:, i - 1, :]) * self.sigmoid_d(
+                self.gates["f"][:, i, :]
+            )
+            grad["W_f"] += np.dot(self.last_input[:, i, :].T, de_f)
+            grad["U_f"] += np.dot(self.outputs[:, i - 1, :].T, de_f)
+            grad["b_f"] += de_f.sum(axis=0)
+
+            de_i = (dhi * self.gates["c"][:, i, :]) * self.sigmoid_d(
+                self.gates["i"][:, i, :]
+            )
+            grad["W_i"] += np.dot(self.last_input[:, i, :].T, de_i)
+            grad["U_i"] += np.dot(self.outputs[:, i - 1, :].T, de_i)
+            grad["b_i"] += de_i.sum(axis=0)
+
+            de_c = (dhi * self.gates["i"][:, i, :]) * self.activation_d(
+                self.gates["c"][:, i, :]
+            )
+            grad["W_c"] += np.dot(self.last_input[:, i, :].T, de_c)
+            grad["U_c"] += np.dot(self.outputs[:, i - 1, :].T, de_c)
+            grad["b_c"] += de_c.sum(axis=0)
+
+            dh_next = dhi * self.gates["f"][:, i, :]
+
+        # TODO: propagate error to the next layer
+
+        # Change actual gradient arrays
+        for k in grad.keys():
+            self._params.update_grad(k, grad[k])
+        return output
+
+    def shape(self, x_shape):
+        if self.return_sequences:
+            return x_shape[0], x_shape[1], self.hidden_dim
+        else:
+            return x_shape[0], self.hidden_dim

mla/neuralnet/layers/recurrent/rnn.py

@@ -0,0 +1,110 @@
+# coding:utf-8
+import autograd.numpy as np
+from autograd import elementwise_grad
+
+from mla.neuralnet.initializations import get_initializer
+from mla.neuralnet.layers import Layer, get_activation, ParamMixin
+from mla.neuralnet.parameters import Parameters
+
+
+class RNN(Layer, ParamMixin):
+    """Vanilla RNN."""
+
+    def __init__(
+        self,
+        hidden_dim,
+        activation="tanh",
+        inner_init="orthogonal",
+        parameters=None,
+        return_sequences=True,
+    ):
+        self.return_sequences = return_sequences
+        self.hidden_dim = hidden_dim
+        self.inner_init = get_initializer(inner_init)
+        self.activation = get_activation(activation)
+        self.activation_d = elementwise_grad(self.activation)
+        if parameters is None:
+            self._params = Parameters()
+        else:
+            self._params = parameters
+        self.last_input = None
+        self.states = None
+        self.hprev = None
+        self.input_dim = None
+
+    def setup(self, x_shape):
+        """
+        Parameters
+        ----------
+        x_shape : np.array(batch size, time steps, input shape)
+        """
+        self.input_dim = x_shape[2]
+
+        # Input -> Hidden
+        self._params["W"] = self._params.init((self.input_dim, self.hidden_dim))
+        # Bias
+        self._params["b"] = np.full((self.hidden_dim,), self._params.initial_bias)
+        # Hidden -> Hidden layer
+        self._params["U"] = self.inner_init((self.hidden_dim, self.hidden_dim))
+
+        # Init gradient arrays
+        self._params.init_grad()
+
+        self.hprev = np.zeros((x_shape[0], self.hidden_dim))
+
+    def forward_pass(self, X):
+        self.last_input = X
+        n_samples, n_timesteps, input_shape = X.shape
+        states = np.zeros((n_samples, n_timesteps + 1, self.hidden_dim))
+        states[:, -1, :] = self.hprev.copy()
+        p = self._params
+
+        for i in range(n_timesteps):
+            states[:, i, :] = np.tanh(
+                np.dot(X[:, i, :], p["W"])
+                + np.dot(states[:, i - 1, :], p["U"])
+                + p["b"]
+            )
+
+        self.states = states
+        self.hprev = states[:, n_timesteps - 1, :].copy()
+        if self.return_sequences:
+            return states[:, 0:-1, :]
+        else:
+            return states[:, -2, :]
+
+    def backward_pass(self, delta):
+        if len(delta.shape) == 2:
+            delta = delta[:, np.newaxis, :]
+        n_samples, n_timesteps, input_shape = delta.shape
+        p = self._params
+
+        # Temporal gradient arrays
+        grad = {k: np.zeros_like(p[k]) for k in p.keys()}
+
+        dh_next = np.zeros((n_samples, input_shape))
+        output = np.zeros((n_samples, n_timesteps, self.input_dim))
+
+        # Backpropagation through time
+        for i in reversed(range(n_timesteps)):
+            dhi = self.activation_d(self.states[:, i, :]) * (delta[:, i, :] + dh_next)
+
+            grad["W"] += np.dot(self.last_input[:, i, :].T, dhi)
+            grad["b"] += delta[:, i, :].sum(axis=0)
+            grad["U"] += np.dot(self.states[:, i - 1, :].T, dhi)
+
+            dh_next = np.dot(dhi, p["U"].T)
+
+            d = np.dot(delta[:, i, :], p["U"].T)
+            output[:, i, :] = np.dot(d, p["W"].T)
+
+        # Change actual gradient arrays
+        for k in grad.keys():
+            self._params.update_grad(k, grad[k])
+        return output
+
+    def shape(self, x_shape):
+        if self.return_sequences:
+            return x_shape[0], x_shape[1], self.hidden_dim
+        else:
+            return x_shape[0], self.hidden_dim

mla/neuralnet/loss.py

@@ -0,0 +1,11 @@
+from ..metrics import mse, logloss, mae, hinge, binary_crossentropy
+
+categorical_crossentropy = logloss
+
+
+def get_loss(name):
+    """Returns loss function by the name."""
+    try:
+        return globals()[name]
+    except KeyError:
+        raise ValueError("Invalid metric function.")

mla/neuralnet/nnet.py

@@ -0,0 +1,181 @@
+import logging
+
+import numpy as np
+from autograd import elementwise_grad
+
+from mla.base import BaseEstimator
+from mla.metrics.metrics import get_metric
+from mla.neuralnet.layers import PhaseMixin
+from mla.neuralnet.loss import get_loss
+from mla.utils import batch_iterator
+
+np.random.seed(9999)
+
+"""
+Architecture inspired from:
+
+    https://github.com/fchollet/keras
+    https://github.com/andersbll/deeppy
+"""
+
+
+class NeuralNet(BaseEstimator):
+    fit_required = False
+
+    def __init__(
+        self,
+        layers,
+        optimizer,
+        loss,
+        max_epochs=10,
+        batch_size=64,
+        metric="mse",
+        shuffle=False,
+        verbose=True,
+    ):
+        self.verbose = verbose
+        self.shuffle = shuffle
+        self.optimizer = optimizer
+
+        self.loss = get_loss(loss)
+
+        # TODO: fix
+        if loss == "categorical_crossentropy":
+            self.loss_grad = lambda actual, predicted: -(actual - predicted)
+        else:
+            self.loss_grad = elementwise_grad(self.loss, 1)
+        self.metric = get_metric(metric)
+        self.layers = layers
+        self.batch_size = batch_size
+        self.max_epochs = max_epochs
+        self._n_layers = 0
+        self.log_metric = True if loss != metric else False
+        self.metric_name = metric
+        self.bprop_entry = self._find_bprop_entry()
+        self.training = False
+        self._initialized = False
+
+    def _setup_layers(self, x_shape):
+        """Initialize model's layers."""
+        x_shape = list(x_shape)
+        x_shape[0] = self.batch_size
+
+        for layer in self.layers:
+            layer.setup(x_shape)
+            x_shape = layer.shape(x_shape)
+
+        self._n_layers = len(self.layers)
+        # Setup optimizer
+        self.optimizer.setup(self)
+        self._initialized = True
+        logging.info("Total parameters: %s" % self.n_params)
+
+    def _find_bprop_entry(self):
+        """Find entry layer for back propagation."""
+
+        if len(self.layers) > 0 and not hasattr(self.layers[-1], "parameters"):
+            return -1
+        return len(self.layers)
+
+    def fit(self, X, y=None):
+        if not self._initialized:
+            self._setup_layers(X.shape)
+
+        if y.ndim == 1:
+            # Reshape vector to matrix
+            y = y[:, np.newaxis]
+        self._setup_input(X, y)
+
+        self.is_training = True
+        # Pass neural network instance to an optimizer
+        self.optimizer.optimize(self)
+        self.is_training = False
+
+    def update(self, X, y):
+        # Forward pass
+        y_pred = self.fprop(X)
+
+        # Backward pass
+        grad = self.loss_grad(y, y_pred)
+        for layer in reversed(self.layers[: self.bprop_entry]):
+            grad = layer.backward_pass(grad)
+        return self.loss(y, y_pred)
+
+    def fprop(self, X):
+        """Forward propagation."""
+        for layer in self.layers:
+            X = layer.forward_pass(X)
+        return X
+
+    def _predict(self, X=None):
+        if not self._initialized:
+            self._setup_layers(X.shape)
+
+        y = []
+        X_batch = batch_iterator(X, self.batch_size)
+        for Xb in X_batch:
+            y.append(self.fprop(Xb))
+        return np.concatenate(y)
+
+    @property
+    def parametric_layers(self):
+        for layer in self.layers:
+            if hasattr(layer, "parameters"):
+                yield layer
+
+    @property
+    def parameters(self):
+        """Returns a list of all parameters."""
+        params = []
+        for layer in self.parametric_layers:
+            params.append(layer.parameters)
+        return params
+
+    def error(self, X=None, y=None):
+        """Calculate an error for given examples."""
+        training_phase = self.is_training
+        if training_phase:
+            # Temporally disable training.
+            # Some layers work differently while training (e.g. Dropout).
+            self.is_training = False
+        if X is None and y is None:
+            y_pred = self._predict(self.X)
+            score = self.metric(self.y, y_pred)
+        else:
+            y_pred = self._predict(X)
+            score = self.metric(y, y_pred)
+        if training_phase:
+            self.is_training = True
+        return score
+
+    @property
+    def is_training(self):
+        return self.training
+
+    @is_training.setter
+    def is_training(self, train):
+        self.training = train
+        for layer in self.layers:
+            if isinstance(layer, PhaseMixin):
+                layer.is_training = train
+
+    def shuffle_dataset(self):
+        """Shuffle rows in the dataset."""
+        n_samples = self.X.shape[0]
+        indices = np.arange(n_samples)
+        np.random.shuffle(indices)
+        self.X = self.X.take(indices, axis=0)
+        self.y = self.y.take(indices, axis=0)
+
+    @property
+    def n_layers(self):
+        """Returns the number of layers."""
+        return self._n_layers
+
+    @property
+    def n_params(self):
+        """Return the number of trainable parameters."""
+        return sum([layer.parameters.n_params for layer in self.parametric_layers])
+
+    def reset(self):
+        self._initialized = False

mla/neuralnet/optimizers.py

@@ -0,0 +1,251 @@
+import logging
+import time
+from collections import defaultdict
+
+import numpy as np
+from tqdm import tqdm
+
+from mla.utils import batch_iterator
+
+"""
+References:
+
+    Gradient descent optimization algorithms  https://ruder.io/optimizing-gradient-descent/
+"""
+
+
+class Optimizer(object):
+    def optimize(self, network):
+        loss_history = []
+        for i in range(network.max_epochs):
+            if network.shuffle:
+                network.shuffle_dataset()
+
+            start_time = time.time()
+            loss = self.train_epoch(network)
+            loss_history.append(loss)
+            if network.verbose:
+                msg = "Epoch:%s, train loss: %s" % (i, loss)
+                if network.log_metric:
+                    msg += ", train %s: %s" % (network.metric_name, network.error())
+                msg += ", elapsed: %s sec." % (time.time() - start_time)
+                logging.info(msg)
+        return loss_history
+
+    def update(self, network):
+        """Performs an update of parameters."""
+        raise NotImplementedError
+
+    def train_epoch(self, network):
+        losses = []
+
+        # Create batch iterator
+        X_batch = batch_iterator(network.X, network.batch_size)
+        y_batch = batch_iterator(network.y, network.batch_size)
+
+        batch = zip(X_batch, y_batch)
+        if network.verbose:
+            batch = tqdm(
+                batch, total=int(np.ceil(network.n_samples / network.batch_size))
+            )
+
+        for X, y in batch:
+            loss = np.mean(network.update(X, y))
+            self.update(network)
+            losses.append(loss)
+
+        epoch_loss = np.mean(losses)
+        return epoch_loss
+
+    def train_batch(self, network, X, y):
+        loss = np.mean(network.update(X, y))
+        self.update(network)
+        return loss
+
+    def setup(self, network):
+        """Creates additional variables.
+        Note: Must be called before optimization process."""
+        raise NotImplementedError
+
+
+class SGD(Optimizer):
+    def __init__(self, learning_rate=0.01, momentum=0.9, decay=0.0, nesterov=False):
+        self.nesterov = nesterov
+        self.decay = decay
+        self.momentum = momentum
+        self.lr = learning_rate
+        self.iteration = 0
+        self.velocity = None
+
+    def update(self, network):
+        lr = self.lr * (1.0 / (1.0 + self.decay * self.iteration))
+
+        for i, layer in enumerate(network.parametric_layers):
+            for n in layer.parameters.keys():
+                # Get gradient values
+                grad = layer.parameters.grad[n]
+                update = self.momentum * self.velocity[i][n] - lr * grad
+                self.velocity[i][n] = update
+                if self.nesterov:
+                    # Adjust using updated velocity
+                    update = self.momentum * self.velocity[i][n] - lr * grad
+                layer.parameters.step(n, update)
+        self.iteration += 1
+
+    def setup(self, network):
+        self.velocity = defaultdict(dict)
+        for i, layer in enumerate(network.parametric_layers):
+            for n in layer.parameters.keys():
+                self.velocity[i][n] = np.zeros_like(layer.parameters[n])
+
+
+class Adagrad(Optimizer):
+    def __init__(self, learning_rate=0.01, epsilon=1e-8):
+        self.eps = epsilon
+        self.lr = learning_rate
+
+    def update(self, network):
+        for i, layer in enumerate(network.parametric_layers):
+            for n in layer.parameters.keys():
+                grad = layer.parameters.grad[n]
+                self.accu[i][n] += grad**2
+                step = self.lr * grad / (np.sqrt(self.accu[i][n]) + self.eps)
+                layer.parameters.step(n, -step)
+
+    def setup(self, network):
+        # Accumulators
+        self.accu = defaultdict(dict)
+        for i, layer in enumerate(network.parametric_layers):
+            for n in layer.parameters.keys():
+                self.accu[i][n] = np.zeros_like(layer.parameters[n])
+
+
+class Adadelta(Optimizer):
+    def __init__(self, learning_rate=1.0, rho=0.95, epsilon=1e-8):
+        self.rho = rho
+        self.eps = epsilon
+        self.lr = learning_rate
+
+    def update(self, network):
+        for i, layer in enumerate(network.parametric_layers):
+            for n in layer.parameters.keys():
+                grad = layer.parameters.grad[n]
+                self.accu[i][n] = (
+                    self.rho * self.accu[i][n] + (1.0 - self.rho) * grad**2
+                )
+                step = (
+                    grad
+                    * np.sqrt(self.d_accu[i][n] + self.eps)
+                    / np.sqrt(self.accu[i][n] + self.eps)
+                )
+
+                layer.parameters.step(n, -step * self.lr)
+                # Update delta accumulator
+                self.d_accu[i][n] = (
+                    self.rho * self.d_accu[i][n] + (1.0 - self.rho) * step**2
+                )
+
+    def setup(self, network):
+        # Accumulators
+        self.accu = defaultdict(dict)
+        self.d_accu = defaultdict(dict)
+        for i, layer in enumerate(network.parametric_layers):
+            for n in layer.parameters.keys():
+                self.accu[i][n] = np.zeros_like(layer.parameters[n])
+                self.d_accu[i][n] = np.zeros_like(layer.parameters[n])
+
+
+class RMSprop(Optimizer):
+    def __init__(self, learning_rate=0.001, rho=0.9, epsilon=1e-8):
+        self.eps = epsilon
+        self.rho = rho
+        self.lr = learning_rate
+
+    def update(self, network):
+        for i, layer in enumerate(network.parametric_layers):
+            for n in layer.parameters.keys():
+                grad = layer.parameters.grad[n]
+                self.accu[i][n] = (self.rho * self.accu[i][n]) + (1.0 - self.rho) * (
+                    grad**2
+                )
+                step = self.lr * grad / (np.sqrt(self.accu[i][n]) + self.eps)
+                layer.parameters.step(n, -step)
+
+    def setup(self, network):
+        # Accumulators
+        self.accu = defaultdict(dict)
+        for i, layer in enumerate(network.parametric_layers):
+            for n in layer.parameters.keys():
+                self.accu[i][n] = np.zeros_like(layer.parameters[n])
+
+
+class Adam(Optimizer):
+    def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8):
+        self.epsilon = epsilon
+        self.beta_2 = beta_2
+        self.beta_1 = beta_1
+        self.lr = learning_rate
+        self.iterations = 0
+        self.t = 1
+
+    def update(self, network):
+        for i, layer in enumerate(network.parametric_layers):
+            for n in layer.parameters.keys():
+                grad = layer.parameters.grad[n]
+                self.ms[i][n] = (self.beta_1 * self.ms[i][n]) + (
+                    1.0 - self.beta_1
+                ) * grad
+                self.vs[i][n] = (self.beta_2 * self.vs[i][n]) + (
+                    1.0 - self.beta_2
+                ) * grad**2
+                lr = (
+                    self.lr
+                    * np.sqrt(1.0 - self.beta_2**self.t)
+                    / (1.0 - self.beta_1**self.t)
+                )
+
+                step = lr * self.ms[i][n] / (np.sqrt(self.vs[i][n]) + self.epsilon)
+                layer.parameters.step(n, -step)
+        self.t += 1
+
+    def setup(self, network):
+        # Accumulators
+        self.ms = defaultdict(dict)
+        self.vs = defaultdict(dict)
+        for i, layer in enumerate(network.parametric_layers):
+            for n in layer.parameters.keys():
+                self.ms[i][n] = np.zeros_like(layer.parameters[n])
+                self.vs[i][n] = np.zeros_like(layer.parameters[n])
+
+
+class Adamax(Optimizer):
+    def __init__(self, learning_rate=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-8):
+        self.epsilon = epsilon
+        self.beta_2 = beta_2
+        self.beta_1 = beta_1
+        self.lr = learning_rate
+        self.t = 1
+
+    def update(self, network):
+        for i, layer in enumerate(network.parametric_layers):
+            for n in layer.parameters.keys():
+                grad = layer.parameters.grad[n]
+                self.ms[i][n] = self.beta_1 * self.ms[i][n] + (1.0 - self.beta_1) * grad
+                self.us[i][n] = np.maximum(self.beta_2 * self.us[i][n], np.abs(grad))
+
+                step = (
+                    self.lr
+                    / (1 - self.beta_1**self.t)
+                    * self.ms[i][n]
+                    / (self.us[i][n] + self.epsilon)
+                )
+                layer.parameters.step(n, -step)
+        self.t += 1
+
+    def setup(self, network):
+        self.ms = defaultdict(dict)
+        self.us = defaultdict(dict)
+        for i, layer in enumerate(network.parametric_layers):
+            for n in layer.parameters.keys():
+                self.ms[i][n] = np.zeros_like(layer.parameters[n])
+                self.us[i][n] = np.zeros_like(layer.parameters[n])

mla/neuralnet/parameters.py

@@ -0,0 +1,98 @@
+# coding:utf-8
+import numpy as np
+
+from mla.neuralnet.initializations import get_initializer
+
+
+class Parameters(object):
+    def __init__(
+        self,
+        init="glorot_uniform",
+        scale=0.5,
+        bias=1.0,
+        regularizers=None,
+        constraints=None,
+    ):
+        """A container for layer's parameters.
+
+        Parameters
+        ----------
+        init : str, default 'glorot_uniform'.
+            The name of the weight initialization function.
+        scale : float, default 0.5
+        bias : float, default 1.0
+            Initial values for bias.
+        regularizers : dict
+            Weight regularizers.
+            >>> {'W' : L2()}
+        constraints : dict
+            Weight constraints.
+            >>> {'b' : MaxNorm()}
+        """
+        if constraints is None:
+            self.constraints = {}
+        else:
+            self.constraints = constraints
+
+        if regularizers is None:
+            self.regularizers = {}
+        else:
+            self.regularizers = regularizers
+
+        self.initial_bias = bias
+        self.scale = scale
+        self.init = get_initializer(init)
+
+        self._params = {}
+        self._grads = {}
+
+    def setup_weights(self, W_shape, b_shape=None):
+        if "W" not in self._params:
+            self._params["W"] = self.init(shape=W_shape, scale=self.scale)
+            if b_shape is None:
+                self._params["b"] = np.full(W_shape[1], self.initial_bias)
+            else:
+                self._params["b"] = np.full(b_shape, self.initial_bias)
+        self.init_grad()
+
+    def init_grad(self):
+        """Init gradient arrays corresponding to each weight array."""
+        for key in self._params.keys():
+            if key not in self._grads:
+                self._grads[key] = np.zeros_like(self._params[key])
+
+    def step(self, name, step):
+        """Increase specific weight by amount of the step parameter."""
+        self._params[name] += step
+
+        if name in self.constraints:
+            self._params[name] = self.constraints[name].clip(self._params[name])
+
+    def update_grad(self, name, value):
+        """Update gradient values."""
+        self._grads[name] = value
+
+        if name in self.regularizers:
+            self._grads[name] += self.regularizers[name](self._params[name])
+
+    @property
+    def n_params(self):
+        """Count the number of parameters in this layer."""
+        return sum([np.prod(self._params[x].shape) for x in self._params.keys()])
+
+    def keys(self):
+        return self._params.keys()
+
+    @property
+    def grad(self):
+        return self._grads
+
+    # Allow access to the fields using dict syntax, e.g. parameters['W']
+    def __getitem__(self, item):
+        if item in self._params:
+            return self._params[item]
+        else:
+            raise ValueError
+
+    def __setitem__(self, key, value):
+        self._params[key] = value

mla/neuralnet/regularizers.py

@@ -0,0 +1,35 @@
+# coding:utf-8
+import numpy as np
+from autograd import elementwise_grad
+
+
+class Regularizer(object):
+    def __init__(self, C=0.01):
+        self.C = C
+        self._grad = elementwise_grad(self._penalty)
+
+    def _penalty(self, weights):
+        raise NotImplementedError()
+
+    def grad(self, weights):
+        return self._grad(weights)
+
+    def __call__(self, weights):
+        return self.grad(weights)
+
+
+class L1(Regularizer):
+    def _penalty(self, weights):
+        return self.C * np.abs(weights)
+
+
+class L2(Regularizer):
+    def _penalty(self, weights):
+        return self.C * weights**2
+
+
+class ElasticNet(Regularizer):
+    """Linear combination of L1 and L2 penalties."""
+
+    def _penalty(self, weights):
+        return 0.5 * self.C * weights**2 + (1.0 - self.C) * np.abs(weights)

mla/neuralnet/tests/test_activations.py

@@ -0,0 +1,21 @@
+import sys
+
+import numpy as np
+
+from mla.neuralnet.activations import *
+
+
+def test_softplus():
+    # np.exp(z_max) will overflow
+    z_max = np.log(sys.float_info.max) + 1.0e10
+    # 1.0 / np.exp(z_min) will overflow
+    z_min = np.log(sys.float_info.min) - 1.0e10
+    inputs = np.array([0.0, 1.0, -1.0, z_min, z_max])
+    # naive implementation of np.log(1 + np.exp(z_max)) will overflow
+    # naive implementation of z + np.log(1 + 1 / np.exp(z_min)) will
+    # throw ZeroDivisionError
+    outputs = np.array(
+        [np.log(2.0), np.log1p(np.exp(1.0)), np.log1p(np.exp(-1.0)), 0.0, z_max]
+    )
+
+    assert np.allclose(outputs, softplus(inputs))

mla/neuralnet/tests/test_optimizers.py

@@ -0,0 +1,72 @@
+from sklearn.datasets import make_classification
+from sklearn.metrics import roc_auc_score
+from sklearn.model_selection import train_test_split
+
+from mla.neuralnet import NeuralNet
+from mla.neuralnet.layers import Dense, Activation, Dropout, Parameters
+from mla.neuralnet.optimizers import *
+from mla.utils import one_hot
+
+
+def clasifier(optimizer):
+    X, y = make_classification(
+        n_samples=1000,
+        n_features=100,
+        n_informative=75,
+        random_state=1111,
+        n_classes=2,
+        class_sep=2.5,
+    )
+    y = one_hot(y)
+
+    X -= np.mean(X, axis=0)
+    X /= np.std(X, axis=0)
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=0.15, random_state=1111
+    )
+
+    model = NeuralNet(
+        layers=[
+            Dense(128, Parameters(init="uniform")),
+            Activation("relu"),
+            Dropout(0.5),
+            Dense(64, Parameters(init="normal")),
+            Activation("relu"),
+            Dense(2),
+            Activation("softmax"),
+        ],
+        loss="categorical_crossentropy",
+        optimizer=optimizer,
+        metric="accuracy",
+        batch_size=64,
+        max_epochs=10,
+    )
+    model.fit(X_train, y_train)
+    predictions = model.predict(X_test)
+    return roc_auc_score(y_test[:, 0], predictions[:, 0])
+
+
+def test_adadelta():
+    assert clasifier(Adadelta()) > 0.9
+
+
+def test_adam():
+    assert clasifier(Adam()) > 0.9
+
+
+def test_adamax():
+    assert clasifier(Adamax()) > 0.9
+
+
+def test_rmsprop():
+    assert clasifier(RMSprop()) > 0.9
+
+
+def test_adagrad():
+    assert clasifier(Adagrad()) > 0.9
+
+
+def test_sgd():
+    assert clasifier(SGD(learning_rate=0.0001)) > 0.9
+    assert clasifier(SGD(learning_rate=0.0001, nesterov=True, momentum=0.9)) > 0.9
+    assert clasifier(SGD(learning_rate=0.0001, nesterov=False, momentum=0.0)) > 0.9