wehub-resource-sync
76 lines
1.9 KiB
Python
76 lines
1.9 KiB
Python
import numpy as np
|
|
|
|
# Collection of activation functions
|
|
# Reference: https://en.wikipedia.org/wiki/Activation_function
|
|
|
|
class Sigmoid():
|
|
def __call__(self, x):
|
|
return 1 / (1 + np.exp(-x))
|
|
|
|
def gradient(self, x):
|
|
return self.__call__(x) * (1 - self.__call__(x))
|
|
|
|
class Softmax():
|
|
def __call__(self, x):
|
|
e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
|
|
return e_x / np.sum(e_x, axis=-1, keepdims=True)
|
|
|
|
def gradient(self, x):
|
|
p = self.__call__(x)
|
|
return p * (1 - p)
|
|
|
|
class TanH():
|
|
def __call__(self, x):
|
|
return 2 / (1 + np.exp(-2*x)) - 1
|
|
|
|
def gradient(self, x):
|
|
return 1 - np.power(self.__call__(x), 2)
|
|
|
|
class ReLU():
|
|
def __call__(self, x):
|
|
return np.where(x >= 0, x, 0)
|
|
|
|
def gradient(self, x):
|
|
return np.where(x >= 0, 1, 0)
|
|
|
|
class LeakyReLU():
|
|
def __init__(self, alpha=0.2):
|
|
self.alpha = alpha
|
|
|
|
def __call__(self, x):
|
|
return np.where(x >= 0, x, self.alpha * x)
|
|
|
|
def gradient(self, x):
|
|
return np.where(x >= 0, 1, self.alpha)
|
|
|
|
class ELU():
|
|
def __init__(self, alpha=0.1):
|
|
self.alpha = alpha
|
|
|
|
def __call__(self, x):
|
|
return np.where(x >= 0.0, x, self.alpha * (np.exp(x) - 1))
|
|
|
|
def gradient(self, x):
|
|
return np.where(x >= 0.0, 1, self.__call__(x) + self.alpha)
|
|
|
|
class SELU():
|
|
# Reference : https://arxiv.org/abs/1706.02515,
|
|
# https://github.com/bioinf-jku/SNNs/blob/master/SelfNormalizingNetworks_MLP_MNIST.ipynb
|
|
def __init__(self):
|
|
self.alpha = 1.6732632423543772848170429916717
|
|
self.scale = 1.0507009873554804934193349852946
|
|
|
|
def __call__(self, x):
|
|
return self.scale * np.where(x >= 0.0, x, self.alpha*(np.exp(x)-1))
|
|
|
|
def gradient(self, x):
|
|
return self.scale * np.where(x >= 0.0, 1, self.alpha * np.exp(x))
|
|
|
|
class SoftPlus():
|
|
def __call__(self, x):
|
|
return np.log(1 + np.exp(x))
|
|
|
|
def gradient(self, x):
|
|
return 1 / (1 + np.exp(-x))
|
|
|