chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,16 @@
|
||||
from prml.nn.optimizer.ada_delta import AdaDelta
|
||||
from prml.nn.optimizer.ada_grad import AdaGrad
|
||||
from prml.nn.optimizer.adam import Adam
|
||||
from prml.nn.optimizer.gradient_ascent import GradientAscent
|
||||
from prml.nn.optimizer.momentum import Momentum
|
||||
from prml.nn.optimizer.rmsprop import RMSProp
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AdaDelta",
|
||||
"AdaGrad",
|
||||
"Adam",
|
||||
"GradientAscent",
|
||||
"Momentum",
|
||||
"RMSProp"
|
||||
]
|
||||
@@ -0,0 +1,31 @@
|
||||
import numpy as np
|
||||
from prml.nn.optimizer.optimizer import Optimizer
|
||||
|
||||
|
||||
class AdaDelta(Optimizer):
|
||||
"""
|
||||
AdaDelta optimizer
|
||||
"""
|
||||
|
||||
def __init__(self, parameter, rho=0.95, epsilon=1e-8):
|
||||
super().__init__(parameter, None)
|
||||
self.rho = rho
|
||||
self.epsilon = epsilon
|
||||
self.mean_squared_deriv = []
|
||||
self.mean_squared_update = []
|
||||
for p in self.parameter:
|
||||
self.mean_squared_deriv.append(np.zeros(p.shape))
|
||||
self.mean_squared_update.append(np.zeros(p.shape))
|
||||
|
||||
def update(self):
|
||||
self.increment_iteration()
|
||||
for p, msd, msu in zip(self.parameter, self.mean_squared_deriv, self.mean_squared_update):
|
||||
if p.grad is None:
|
||||
continue
|
||||
grad = p.grad
|
||||
msd *= self.rho
|
||||
msd += (1 - self.rho) * grad ** 2
|
||||
delta = np.sqrt((msu + self.epsilon) / (msd + self.epsilon)) * grad
|
||||
msu *= self.rho
|
||||
msu *= (1 - self.rho) * delta ** 2
|
||||
p.value += delta
|
||||
@@ -0,0 +1,32 @@
|
||||
import numpy as np
|
||||
from prml.nn.optimizer.optimizer import Optimizer
|
||||
|
||||
|
||||
class AdaGrad(Optimizer):
|
||||
"""
|
||||
AdaGrad optimizer
|
||||
initialization
|
||||
G = 0
|
||||
update rule
|
||||
G += gradient ** 2
|
||||
param -= learning_rate * gradient / sqrt(G + eps)
|
||||
"""
|
||||
|
||||
def __init__(self, parameter, learning_rate=0.001, epsilon=1e-8):
|
||||
super().__init__(parameter, learning_rate)
|
||||
self.epsilon = epsilon
|
||||
self.G = []
|
||||
for p in self.parameter:
|
||||
self.G.append(np.zeros(p.shape))
|
||||
|
||||
def update(self):
|
||||
"""
|
||||
update parameters
|
||||
"""
|
||||
self.increment_iteration()
|
||||
for p, G in zip(self.parameter, self.G):
|
||||
if p.grad is None:
|
||||
continue
|
||||
grad = p.grad
|
||||
G += grad ** 2
|
||||
p.value += self.learning_rate * grad / (np.sqrt(G) + self.epsilon)
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
import numpy as np
|
||||
from prml.nn.optimizer.optimizer import Optimizer
|
||||
|
||||
|
||||
class Adam(Optimizer):
|
||||
"""
|
||||
Adam optimizer
|
||||
initialization
|
||||
m1 = 0 (Initial 1st moment of gradient)
|
||||
m2 = 0 (Initial 2nd moment of gradient)
|
||||
n_iter = 0
|
||||
update rule
|
||||
n_iter += 1
|
||||
learning_rate *= sqrt(1 - beta2^n) / (1 - beta1^n)
|
||||
m1 = beta1 * m1 + (1 - beta1) * gradient
|
||||
m2 = beta2 * m2 + (1 - beta2) * gradient^2
|
||||
param += learning_rate * m1 / (sqrt(m2) + epsilon)
|
||||
"""
|
||||
|
||||
def __init__(self, parameter, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8):
|
||||
"""
|
||||
construct Adam optimizer
|
||||
Parameters
|
||||
----------
|
||||
parameters : list
|
||||
list of parameters to be optimized
|
||||
learning_rate : float
|
||||
beta1 : float
|
||||
exponential decay rate for the 1st moment
|
||||
beta2 : float
|
||||
exponential decay rate for the 2nd moment
|
||||
epsilon : float
|
||||
small constant to be added to denominator for numerical stability
|
||||
Attributes
|
||||
----------
|
||||
n_iter : int
|
||||
number of iterations performed
|
||||
moment1 : dict
|
||||
1st moment of each learnable parameter
|
||||
moment2 : dict
|
||||
2nd moment of each learnable parameter
|
||||
"""
|
||||
super().__init__(parameter, learning_rate)
|
||||
self.beta1 = beta1
|
||||
self.beta2 = beta2
|
||||
self.epsilon = epsilon
|
||||
self.moment1 = []
|
||||
self.moment2 = []
|
||||
for p in self.parameter:
|
||||
self.moment1.append(np.zeros(p.shape))
|
||||
self.moment2.append(np.zeros(p.shape))
|
||||
|
||||
def update(self):
|
||||
"""
|
||||
update parameter of the neural network
|
||||
"""
|
||||
self.increment_iteration()
|
||||
lr = (
|
||||
self.learning_rate
|
||||
* (1 - self.beta2 ** self.n_iter) ** 0.5
|
||||
/ (1 - self.beta1 ** self.n_iter))
|
||||
for p, m1, m2 in zip(self.parameter, self.moment1, self.moment2):
|
||||
if p.grad is None:
|
||||
continue
|
||||
m1 += (1 - self.beta1) * (p.grad - m1)
|
||||
m2 += (1 - self.beta2) * (p.grad ** 2 - m2)
|
||||
p.value += lr * m1 / (np.sqrt(m2) + self.epsilon)
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
import numpy as np
|
||||
from prml.nn.optimizer.optimizer import Optimizer
|
||||
|
||||
|
||||
class Eve(Optimizer):
|
||||
"""
|
||||
Eve optimizer
|
||||
|
||||
initialization
|
||||
m1 = 0 (initial 1st moment of gradient)
|
||||
m2 = 0 (initial 2nd moment of gradient)
|
||||
n_iter = 0
|
||||
|
||||
update rule
|
||||
n_iter += 1
|
||||
learning
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
network,
|
||||
learning_rate=0.001,
|
||||
beta1=0.9,
|
||||
beta2=0.999,
|
||||
beta3=0.999,
|
||||
lower_threshold=0.1,
|
||||
upper_threshold=10.,
|
||||
epsilon=1e-8):
|
||||
"""
|
||||
construct Eve optimizer
|
||||
|
||||
Parameters
|
||||
----------
|
||||
network : Network
|
||||
neural network to be optmized
|
||||
learning_rate : float
|
||||
beta1 : float
|
||||
exponential decay rate for the 1st moment
|
||||
beta2 : float
|
||||
exponential decay rate for the 2nd moment
|
||||
beta3 : float
|
||||
exponential decay rate for computing relative change
|
||||
lower_threshold : float
|
||||
lower threshold for relative change
|
||||
upper_threshold : float
|
||||
upper threshold for relative change
|
||||
epsilon : float
|
||||
small constant to be added to denominator for numerical stability
|
||||
|
||||
Attributes
|
||||
----------
|
||||
n_iter : int
|
||||
number of iterations performed
|
||||
moment1 : dict
|
||||
1st moment of each parameter
|
||||
moment2 : dict
|
||||
2nd moment of each parameter
|
||||
"""
|
||||
super().__init__(network, learning_rate)
|
||||
self.beta1 = beta1
|
||||
self.beta2 = beta2
|
||||
self.beta3 = beta3
|
||||
self.lower_threshold = lower_threshold
|
||||
self.upper_threshold = upper_threshold
|
||||
self.epsilon = epsilon
|
||||
self.moment1 = {}
|
||||
self.moment2 = {}
|
||||
self.f = 1.
|
||||
self.d = 1.
|
||||
for key, param in self.params.items():
|
||||
self.moment1[key] = np.zeros(param.shape)
|
||||
self.moment2[key] = np.zeros(param.shape)
|
||||
|
||||
def update(self, loss):
|
||||
loss = float(loss)
|
||||
self.increment_iteration()
|
||||
if self.n_iter > 1:
|
||||
if loss > self.f:
|
||||
delta = self.lower_threshold + 1
|
||||
Delta = self.upper_threshold + 1
|
||||
else:
|
||||
delta = 1 / (self.upper_threshold + 1)
|
||||
Delta = 1 / (self.lower_threshold + 1)
|
||||
c = min(max(delta, loss / self.f), Delta)
|
||||
f = c * self.f
|
||||
r = abs(f - self.f) / min(f, self.f)
|
||||
self.d = self.beta3 * self.d * (1 - self.beta3) * r
|
||||
self.f = f
|
||||
else:
|
||||
self.f = loss
|
||||
self.d = 1
|
||||
lr = (
|
||||
self.learning_rate
|
||||
* (1 - self.beta2 ** self.n_iter) ** 0.5
|
||||
/ (1 - self.beta1 ** self.n_iter)
|
||||
)
|
||||
for key, param in self.params.items():
|
||||
m1 = self.moment1[key]
|
||||
m2 = self.moment2[key]
|
||||
m1 += (1 - self.beta1) * (param.grad - m1)
|
||||
m2 += (1 - self.beta2) * (param.grad ** 2 - m2)
|
||||
param.value -= lr * m1 / (self.d * np.sqrt(m2) + self.epsilon)
|
||||
@@ -0,0 +1,18 @@
|
||||
from prml.nn.optimizer.optimizer import Optimizer
|
||||
|
||||
|
||||
class GradientAscent(Optimizer):
|
||||
"""
|
||||
gradient ascent optimizer
|
||||
parameter += learning_rate * gradient
|
||||
"""
|
||||
|
||||
def update(self):
|
||||
"""
|
||||
update parameters to be optimized
|
||||
"""
|
||||
self.increment_iteration()
|
||||
for p in self.parameter:
|
||||
if p.grad is None:
|
||||
continue
|
||||
p.value += self.learning_rate * p.grad
|
||||
@@ -0,0 +1,29 @@
|
||||
import numpy as np
|
||||
from prml.nn.optimizer.optimizer import Optimizer
|
||||
|
||||
|
||||
class Momentum(Optimizer):
|
||||
"""
|
||||
Momentum optimizer
|
||||
initialization
|
||||
v = 0
|
||||
update rule
|
||||
v = v * momentum - learning_rate * gradient
|
||||
param += v
|
||||
"""
|
||||
|
||||
def __init__(self, parameter, learning_rate, momentum=0.9):
|
||||
super().__init__(parameter, learning_rate)
|
||||
self.momentum = momentum
|
||||
self.inertia = []
|
||||
for p in self.parameter:
|
||||
self.inertia.append(np.zeros(p.shape))
|
||||
|
||||
def update(self):
|
||||
self.increment_iteration()
|
||||
for p, inertia in zip(self.parameter, self.inertia):
|
||||
if p.grad is None:
|
||||
continue
|
||||
inertia *= self.momentum
|
||||
inertia -= self.learning_rate * p.grad
|
||||
p.value += inertia
|
||||
@@ -0,0 +1,52 @@
|
||||
from prml.nn.network import Network
|
||||
|
||||
|
||||
class Optimizer(object):
|
||||
"""
|
||||
Optimizer to train neural network
|
||||
"""
|
||||
|
||||
def __init__(self, parameter, learning_rate):
|
||||
"""
|
||||
construct optimizer
|
||||
Parameters
|
||||
----------
|
||||
parameter : list, dict, Network
|
||||
list of parameter to be optimized
|
||||
learning_rate : float
|
||||
update rate of parameter to be optimized
|
||||
Attributes
|
||||
----------
|
||||
n_iter : int
|
||||
number of iterations performed
|
||||
"""
|
||||
if isinstance(parameter, Network):
|
||||
parameter = parameter.parameter
|
||||
if isinstance(parameter, dict):
|
||||
parameter = list(parameter.values())
|
||||
self.parameter = parameter
|
||||
self.learning_rate = learning_rate
|
||||
self.n_iter = 0
|
||||
|
||||
def cleargrad(self):
|
||||
for p in self.parameter:
|
||||
p.cleargrad()
|
||||
|
||||
def set_decay(self, decay_rate, decay_step):
|
||||
"""
|
||||
set exponential decay parameters
|
||||
Parameters
|
||||
----------
|
||||
decay_rate : float
|
||||
dacay rate of the learning rate
|
||||
decay_step : int
|
||||
steps to decay the learning rate
|
||||
"""
|
||||
self.decay_rate = decay_rate
|
||||
self.decay_step = decay_step
|
||||
|
||||
def increment_iteration(self):
|
||||
self.n_iter += 1
|
||||
if hasattr(self, "decay_rate"):
|
||||
if self.n_iter % self.decay_step == 0:
|
||||
self.learning_rate *= self.decay_rate
|
||||
@@ -0,0 +1,36 @@
|
||||
import numpy as np
|
||||
from prml.nn.optimizer.optimizer import Optimizer
|
||||
|
||||
|
||||
class RMSProp(Optimizer):
|
||||
"""
|
||||
RMSProp optimizer
|
||||
initial
|
||||
msg = 0
|
||||
update rule
|
||||
msg = rho * msg + (1 - rho) * gradient ** 2
|
||||
param -= learning_rate * gradient / (sqrt(msg) + eps)
|
||||
"""
|
||||
|
||||
def __init__(self, parameter, learning_rate=1e-3, rho=0.9, epsilon=1e-8):
|
||||
super().__init__(parameter, learning_rate)
|
||||
self.rho = rho
|
||||
self.epsilon = epsilon
|
||||
self.mean_squared_grad = []
|
||||
for p in self.parameter:
|
||||
self.mean_squared_grad.append(np.zeros(p.shape))
|
||||
|
||||
def update(self):
|
||||
"""
|
||||
update parameters
|
||||
"""
|
||||
self.increment_iteration()
|
||||
for p, msg in zip(self.parameter, self.mean_squared_grad):
|
||||
if p.grad is None:
|
||||
continue
|
||||
grad = p.grad
|
||||
msg *= self.rho
|
||||
msg += (1 - self.rho) * grad ** 2
|
||||
p.value -= (
|
||||
self.learning_rate * grad / (np.sqrt(msg) + self.epsilon)
|
||||
)
|
||||
Reference in New Issue
Block a user