chore: import upstream snapshot with attribution

mla/neuralnet/tests/test_activations.py

@@ -0,0 +1,21 @@
+import sys
+
+import numpy as np
+
+from mla.neuralnet.activations import *
+
+
+def test_softplus():
+    # np.exp(z_max) will overflow
+    z_max = np.log(sys.float_info.max) + 1.0e10
+    # 1.0 / np.exp(z_min) will overflow
+    z_min = np.log(sys.float_info.min) - 1.0e10
+    inputs = np.array([0.0, 1.0, -1.0, z_min, z_max])
+    # naive implementation of np.log(1 + np.exp(z_max)) will overflow
+    # naive implementation of z + np.log(1 + 1 / np.exp(z_min)) will
+    # throw ZeroDivisionError
+    outputs = np.array(
+        [np.log(2.0), np.log1p(np.exp(1.0)), np.log1p(np.exp(-1.0)), 0.0, z_max]
+    )
+
+    assert np.allclose(outputs, softplus(inputs))

mla/neuralnet/tests/test_optimizers.py

@@ -0,0 +1,72 @@
+from sklearn.datasets import make_classification
+from sklearn.metrics import roc_auc_score
+from sklearn.model_selection import train_test_split
+
+from mla.neuralnet import NeuralNet
+from mla.neuralnet.layers import Dense, Activation, Dropout, Parameters
+from mla.neuralnet.optimizers import *
+from mla.utils import one_hot
+
+
+def clasifier(optimizer):
+    X, y = make_classification(
+        n_samples=1000,
+        n_features=100,
+        n_informative=75,
+        random_state=1111,
+        n_classes=2,
+        class_sep=2.5,
+    )
+    y = one_hot(y)
+
+    X -= np.mean(X, axis=0)
+    X /= np.std(X, axis=0)
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=0.15, random_state=1111
+    )
+
+    model = NeuralNet(
+        layers=[
+            Dense(128, Parameters(init="uniform")),
+            Activation("relu"),
+            Dropout(0.5),
+            Dense(64, Parameters(init="normal")),
+            Activation("relu"),
+            Dense(2),
+            Activation("softmax"),
+        ],
+        loss="categorical_crossentropy",
+        optimizer=optimizer,
+        metric="accuracy",
+        batch_size=64,
+        max_epochs=10,
+    )
+    model.fit(X_train, y_train)
+    predictions = model.predict(X_test)
+    return roc_auc_score(y_test[:, 0], predictions[:, 0])
+
+
+def test_adadelta():
+    assert clasifier(Adadelta()) > 0.9
+
+
+def test_adam():
+    assert clasifier(Adam()) > 0.9
+
+
+def test_adamax():
+    assert clasifier(Adamax()) > 0.9
+
+
+def test_rmsprop():
+    assert clasifier(RMSprop()) > 0.9
+
+
+def test_adagrad():
+    assert clasifier(Adagrad()) > 0.9
+
+
+def test_sgd():
+    assert clasifier(SGD(learning_rate=0.0001)) > 0.9
+    assert clasifier(SGD(learning_rate=0.0001, nesterov=True, momentum=0.9)) > 0.9
+    assert clasifier(SGD(learning_rate=0.0001, nesterov=False, momentum=0.0)) > 0.9