wehub-resource-sync
76 lines
1.7 KiB
Python
76 lines
1.7 KiB
Python
import numpy as np
|
|
|
|
"""
|
|
References:
|
|
http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf
|
|
|
|
"""
|
|
|
|
|
|
def normal(shape, scale=0.5):
|
|
return np.random.normal(size=shape, scale=scale)
|
|
|
|
|
|
def uniform(shape, scale=0.5):
|
|
return np.random.uniform(size=shape, low=-scale, high=scale)
|
|
|
|
|
|
def zero(shape, **kwargs):
|
|
return np.zeros(shape)
|
|
|
|
|
|
def one(shape, **kwargs):
|
|
return np.ones(shape)
|
|
|
|
|
|
def orthogonal(shape, scale=0.5):
|
|
flat_shape = (shape[0], np.prod(shape[1:]))
|
|
array = np.random.normal(size=flat_shape)
|
|
u, _, v = np.linalg.svd(array, full_matrices=False)
|
|
array = u if u.shape == flat_shape else v
|
|
return np.reshape(array * scale, shape)
|
|
|
|
|
|
def _glorot_fan(shape):
|
|
assert len(shape) >= 2
|
|
|
|
if len(shape) == 4:
|
|
receptive_field_size = np.prod(shape[2:])
|
|
fan_in = shape[1] * receptive_field_size
|
|
fan_out = shape[0] * receptive_field_size
|
|
else:
|
|
fan_in, fan_out = shape[:2]
|
|
return float(fan_in), float(fan_out)
|
|
|
|
|
|
def glorot_normal(shape, **kwargs):
|
|
fan_in, fan_out = _glorot_fan(shape)
|
|
s = np.sqrt(2.0 / (fan_in + fan_out))
|
|
return normal(shape, s)
|
|
|
|
|
|
def glorot_uniform(shape, **kwargs):
|
|
fan_in, fan_out = _glorot_fan(shape)
|
|
s = np.sqrt(6.0 / (fan_in + fan_out))
|
|
return uniform(shape, s)
|
|
|
|
|
|
def he_normal(shape, **kwargs):
|
|
fan_in, fan_out = _glorot_fan(shape)
|
|
s = np.sqrt(2.0 / fan_in)
|
|
return normal(shape, s)
|
|
|
|
|
|
def he_uniform(shape, **kwargs):
|
|
fan_in, fan_out = _glorot_fan(shape)
|
|
s = np.sqrt(6.0 / fan_in)
|
|
return uniform(shape, s)
|
|
|
|
|
|
def get_initializer(name):
|
|
"""Returns initialization function by the name."""
|
|
try:
|
|
return globals()[name]
|
|
except Exception:
|
|
raise ValueError("Invalid initialization function.")
|