chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
+19
@@ -0,0 +1,19 @@
|
||||
from prml.kernel.polynomial import PolynomialKernel
|
||||
from prml.kernel.rbf import RBF
|
||||
|
||||
from prml.kernel.gaussian_process_classifier import GaussianProcessClassifier
|
||||
from prml.kernel.gaussian_process_regressor import GaussianProcessRegressor
|
||||
from prml.kernel.relevance_vector_classifier import RelevanceVectorClassifier
|
||||
from prml.kernel.relevance_vector_regressor import RelevanceVectorRegressor
|
||||
from prml.kernel.support_vector_classifier import SupportVectorClassifier
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PolynomialKernel",
|
||||
"RBF",
|
||||
"GaussianProcessClassifier",
|
||||
"GaussianProcessRegressor",
|
||||
"RelevanceVectorClassifier",
|
||||
"RelevanceVectorRegressor",
|
||||
"SupportVectorClassifier"
|
||||
]
|
||||
@@ -0,0 +1,37 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
class GaussianProcessClassifier(object):
|
||||
|
||||
def __init__(self, kernel, noise_level=1e-4):
|
||||
"""
|
||||
construct gaussian process classifier
|
||||
|
||||
Parameters
|
||||
----------
|
||||
kernel
|
||||
kernel function to be used to compute Gram matrix
|
||||
noise_level : float
|
||||
parameter to ensure the matrix to be positive
|
||||
"""
|
||||
self.kernel = kernel
|
||||
self.noise_level = noise_level
|
||||
|
||||
def _sigmoid(self, a):
|
||||
return np.tanh(a * 0.5) * 0.5 + 0.5
|
||||
|
||||
def fit(self, X, t):
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
self.X = X
|
||||
self.t = t
|
||||
Gram = self.kernel(X, X)
|
||||
self.covariance = Gram + np.eye(len(Gram)) * self.noise_level
|
||||
self.precision = np.linalg.inv(self.covariance)
|
||||
|
||||
def predict(self, X):
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
K = self.kernel(X, self.X)
|
||||
a_mean = K @ self.precision @ self.t
|
||||
return self._sigmoid(a_mean)
|
||||
@@ -0,0 +1,105 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
class GaussianProcessRegressor(object):
|
||||
|
||||
def __init__(self, kernel, beta=1.):
|
||||
"""
|
||||
construct gaussian process regressor
|
||||
|
||||
Parameters
|
||||
----------
|
||||
kernel
|
||||
kernel function
|
||||
beta : float
|
||||
precision parameter of observation noise
|
||||
"""
|
||||
self.kernel = kernel
|
||||
self.beta = beta
|
||||
|
||||
def fit(self, X, t, iter_max=0, learning_rate=0.1):
|
||||
"""
|
||||
maximum likelihood estimation of parameters in kernel function
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : ndarray (sample_size, n_features)
|
||||
input
|
||||
t : ndarray (sample_size,)
|
||||
corresponding target
|
||||
iter_max : int
|
||||
maximum number of iterations updating hyperparameters
|
||||
learning_rate : float
|
||||
updation coefficient
|
||||
|
||||
Attributes
|
||||
----------
|
||||
covariance : ndarray (sample_size, sample_size)
|
||||
variance covariance matrix of gaussian process
|
||||
precision : ndarray (sample_size, sample_size)
|
||||
precision matrix of gaussian process
|
||||
|
||||
Returns
|
||||
-------
|
||||
log_likelihood_list : list
|
||||
list of log likelihood value at each iteration
|
||||
"""
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
log_likelihood_list = [-np.Inf]
|
||||
self.X = X
|
||||
self.t = t
|
||||
I = np.eye(len(X))
|
||||
Gram = self.kernel(X, X)
|
||||
self.covariance = Gram + I / self.beta
|
||||
self.precision = np.linalg.inv(self.covariance)
|
||||
for i in range(iter_max):
|
||||
gradients = self.kernel.derivatives(X, X)
|
||||
updates = np.array(
|
||||
[-np.trace(self.precision.dot(grad)) + t.dot(self.precision.dot(grad).dot(self.precision).dot(t)) for grad in gradients])
|
||||
for j in range(iter_max):
|
||||
self.kernel.update_parameters(learning_rate * updates)
|
||||
Gram = self.kernel(X, X)
|
||||
self.covariance = Gram + I / self.beta
|
||||
self.precision = np.linalg.inv(self.covariance)
|
||||
log_like = self.log_likelihood()
|
||||
if log_like > log_likelihood_list[-1]:
|
||||
log_likelihood_list.append(log_like)
|
||||
break
|
||||
else:
|
||||
self.kernel.update_parameters(-learning_rate * updates)
|
||||
learning_rate *= 0.9
|
||||
log_likelihood_list.pop(0)
|
||||
return log_likelihood_list
|
||||
|
||||
def log_likelihood(self):
|
||||
return -0.5 * (
|
||||
np.linalg.slogdet(self.covariance)[1]
|
||||
+ self.t @ self.precision @ self.t
|
||||
+ len(self.t) * np.log(2 * np.pi))
|
||||
|
||||
def predict(self, X, with_error=False):
|
||||
"""
|
||||
mean of the gaussian process
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : ndarray (sample_size, n_features)
|
||||
input
|
||||
|
||||
Returns
|
||||
-------
|
||||
mean : ndarray (sample_size,)
|
||||
predictions of corresponding inputs
|
||||
"""
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
K = self.kernel(X, self.X)
|
||||
mean = K @ self.precision @ self.t
|
||||
if with_error:
|
||||
var = (
|
||||
self.kernel(X, X, False)
|
||||
+ 1 / self.beta
|
||||
- np.sum(K @ self.precision * K, axis=1))
|
||||
return mean.ravel(), np.sqrt(var.ravel())
|
||||
return mean
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
class Kernel(object):
|
||||
"""
|
||||
Base class for kernel function
|
||||
"""
|
||||
|
||||
def _pairwise(self, x, y):
|
||||
"""
|
||||
all pairs of x and y
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x : (sample_size, n_features)
|
||||
input
|
||||
y : (sample_size, n_features)
|
||||
another input
|
||||
|
||||
Returns
|
||||
-------
|
||||
output : tuple
|
||||
two array with shape (sample_size, sample_size, n_features)
|
||||
"""
|
||||
return (
|
||||
np.tile(x, (len(y), 1, 1)).transpose(1, 0, 2),
|
||||
np.tile(y, (len(x), 1, 1))
|
||||
)
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
import numpy as np
|
||||
from prml.kernel.kernel import Kernel
|
||||
|
||||
|
||||
class PolynomialKernel(Kernel):
|
||||
"""
|
||||
Polynomial kernel
|
||||
k(x,y) = (x @ y + c)^M
|
||||
"""
|
||||
|
||||
def __init__(self, degree=2, const=0.):
|
||||
"""
|
||||
construct Polynomial kernel
|
||||
|
||||
Parameters
|
||||
----------
|
||||
const : float
|
||||
a constant to be added
|
||||
degree : int
|
||||
degree of polynomial order
|
||||
"""
|
||||
self.const = const
|
||||
self.degree = degree
|
||||
|
||||
def __call__(self, x, y, pairwise=True):
|
||||
"""
|
||||
calculate pairwise polynomial kernel
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x : (..., ndim) ndarray
|
||||
input
|
||||
y : (..., ndim) ndarray
|
||||
another input with the same shape
|
||||
|
||||
Returns
|
||||
-------
|
||||
output : ndarray
|
||||
polynomial kernel
|
||||
"""
|
||||
if pairwise:
|
||||
x, y = self._pairwise(x, y)
|
||||
return (np.sum(x * y, axis=-1) + self.const) ** self.degree
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
import numpy as np
|
||||
from prml.kernel.kernel import Kernel
|
||||
|
||||
|
||||
class RBF(Kernel):
|
||||
|
||||
def __init__(self, params):
|
||||
"""
|
||||
construct Radial basis kernel function
|
||||
|
||||
Parameters
|
||||
----------
|
||||
params : (ndim + 1,) ndarray
|
||||
parameters of radial basis function
|
||||
|
||||
Attributes
|
||||
----------
|
||||
ndim : int
|
||||
dimension of expected input data
|
||||
"""
|
||||
assert params.ndim == 1
|
||||
self.params = params
|
||||
self.ndim = len(params) - 1
|
||||
|
||||
def __call__(self, x, y, pairwise=True):
|
||||
"""
|
||||
calculate radial basis function
|
||||
k(x, y) = c0 * exp(-0.5 * c1 * (x1 - y1) ** 2 ...)
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x : ndarray [..., ndim]
|
||||
input of this kernel function
|
||||
y : ndarray [..., ndim]
|
||||
another input
|
||||
|
||||
Returns
|
||||
-------
|
||||
output : ndarray
|
||||
output of this radial basis function
|
||||
"""
|
||||
assert x.shape[-1] == self.ndim
|
||||
assert y.shape[-1] == self.ndim
|
||||
if pairwise:
|
||||
x, y = self._pairwise(x, y)
|
||||
d = self.params[1:] * (x - y) ** 2
|
||||
return self.params[0] * np.exp(-0.5 * np.sum(d, axis=-1))
|
||||
|
||||
def derivatives(self, x, y, pairwise=True):
|
||||
if pairwise:
|
||||
x, y = self._pairwise(x, y)
|
||||
d = self.params[1:] * (x - y) ** 2
|
||||
delta = np.exp(-0.5 * np.sum(d, axis=-1))
|
||||
deltas = -0.5 * (x - y) ** 2 * (delta * self.params[0])[:, :, None]
|
||||
return np.concatenate((np.expand_dims(delta, 0), deltas.T))
|
||||
|
||||
def update_parameters(self, updates):
|
||||
self.params += updates
|
||||
@@ -0,0 +1,122 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
class RelevanceVectorClassifier(object):
|
||||
|
||||
def __init__(self, kernel, alpha=1.):
|
||||
"""
|
||||
construct relevance vector classifier
|
||||
|
||||
Parameters
|
||||
----------
|
||||
kernel : Kernel
|
||||
kernel function to compute components of feature vectors
|
||||
alpha : float
|
||||
initial precision of prior weight distribution
|
||||
"""
|
||||
self.kernel = kernel
|
||||
self.alpha = alpha
|
||||
|
||||
def _sigmoid(self, a):
|
||||
return np.tanh(a * 0.5) * 0.5 + 0.5
|
||||
|
||||
def _map_estimate(self, X, t, w, n_iter=10):
|
||||
for _ in range(n_iter):
|
||||
y = self._sigmoid(X @ w)
|
||||
g = X.T @ (y - t) + self.alpha * w
|
||||
H = (X.T * y * (1 - y)) @ X + np.diag(self.alpha)
|
||||
w -= np.linalg.solve(H, g)
|
||||
return w, np.linalg.inv(H)
|
||||
|
||||
def fit(self, X, t, iter_max=100):
|
||||
"""
|
||||
maximize evidence with respect ot hyperparameter
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features) ndarray
|
||||
input
|
||||
t : (sample_size,) ndarray
|
||||
corresponding target
|
||||
iter_max : int
|
||||
maximum number of iterations
|
||||
|
||||
Attributes
|
||||
----------
|
||||
X : (N, n_features) ndarray
|
||||
relevance vector
|
||||
t : (N,) ndarray
|
||||
corresponding target
|
||||
alpha : (N,) ndarray
|
||||
hyperparameter for each weight or training sample
|
||||
cov : (N, N) ndarray
|
||||
covariance matrix of weight
|
||||
mean : (N,) ndarray
|
||||
mean of each weight
|
||||
"""
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
assert X.ndim == 2
|
||||
assert t.ndim == 1
|
||||
Phi = self.kernel(X, X)
|
||||
N = len(t)
|
||||
self.alpha = np.zeros(N) + self.alpha
|
||||
mean = np.zeros(N)
|
||||
for _ in range(iter_max):
|
||||
param = np.copy(self.alpha)
|
||||
mean, cov = self._map_estimate(Phi, t, mean, 10)
|
||||
gamma = 1 - self.alpha * np.diag(cov)
|
||||
self.alpha = gamma / np.square(mean)
|
||||
np.clip(self.alpha, 0, 1e10, out=self.alpha)
|
||||
if np.allclose(param, self.alpha):
|
||||
break
|
||||
mask = self.alpha < 1e8
|
||||
self.X = X[mask]
|
||||
self.t = t[mask]
|
||||
self.alpha = self.alpha[mask]
|
||||
Phi = self.kernel(self.X, self.X)
|
||||
mean = mean[mask]
|
||||
self.mean, self.covariance = self._map_estimate(Phi, self.t, mean, 100)
|
||||
|
||||
def predict(self, X):
|
||||
"""
|
||||
predict class label
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features)
|
||||
input
|
||||
|
||||
Returns
|
||||
-------
|
||||
label : (sample_size,) ndarray
|
||||
predicted label
|
||||
"""
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
assert X.ndim == 2
|
||||
phi = self.kernel(X, self.X)
|
||||
label = (phi @ self.mean > 0).astype(np.int)
|
||||
return label
|
||||
|
||||
def predict_proba(self, X):
|
||||
"""
|
||||
probability of input belonging class one
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features) ndarray
|
||||
input
|
||||
|
||||
Returns
|
||||
-------
|
||||
proba : (sample_size,) ndarray
|
||||
probability of predictive distribution p(C1|x)
|
||||
"""
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
assert X.ndim == 2
|
||||
phi = self.kernel(X, self.X)
|
||||
mu_a = phi @ self.mean
|
||||
var_a = np.sum(phi @ self.covariance * phi, axis=1)
|
||||
return self._sigmoid(mu_a / np.sqrt(1 + np.pi * var_a / 8))
|
||||
@@ -0,0 +1,102 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
class RelevanceVectorRegressor(object):
|
||||
|
||||
def __init__(self, kernel, alpha=1., beta=1.):
|
||||
"""
|
||||
construct relevance vector regressor
|
||||
|
||||
Parameters
|
||||
----------
|
||||
kernel : Kernel
|
||||
kernel function to compute components of feature vectors
|
||||
alpha : float
|
||||
initial precision of prior weight distribution
|
||||
beta : float
|
||||
precision of observation
|
||||
"""
|
||||
self.kernel = kernel
|
||||
self.alpha = alpha
|
||||
self.beta = beta
|
||||
|
||||
def fit(self, X, t, iter_max=1000):
|
||||
"""
|
||||
maximize evidence with respect to hyperparameter
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features) ndarray
|
||||
input
|
||||
t : (sample_size,) ndarray
|
||||
corresponding target
|
||||
iter_max : int
|
||||
maximum number of iterations
|
||||
|
||||
Attributes
|
||||
-------
|
||||
X : (N, n_features) ndarray
|
||||
relevance vector
|
||||
t : (N,) ndarray
|
||||
corresponding target
|
||||
alpha : (N,) ndarray
|
||||
hyperparameter for each weight or training sample
|
||||
cov : (N, N) ndarray
|
||||
covariance matrix of weight
|
||||
mean : (N,) ndarray
|
||||
mean of each weight
|
||||
"""
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
assert X.ndim == 2
|
||||
assert t.ndim == 1
|
||||
N = len(t)
|
||||
Phi = self.kernel(X, X)
|
||||
self.alpha = np.zeros(N) + self.alpha
|
||||
for _ in range(iter_max):
|
||||
params = np.hstack([self.alpha, self.beta])
|
||||
precision = np.diag(self.alpha) + self.beta * Phi.T @ Phi
|
||||
covariance = np.linalg.inv(precision)
|
||||
mean = self.beta * covariance @ Phi.T @ t
|
||||
gamma = 1 - self.alpha * np.diag(covariance)
|
||||
self.alpha = gamma / np.square(mean)
|
||||
np.clip(self.alpha, 0, 1e10, out=self.alpha)
|
||||
self.beta = (N - np.sum(gamma)) / np.sum((t - Phi.dot(mean)) ** 2)
|
||||
if np.allclose(params, np.hstack([self.alpha, self.beta])):
|
||||
break
|
||||
mask = self.alpha < 1e9
|
||||
self.X = X[mask]
|
||||
self.t = t[mask]
|
||||
self.alpha = self.alpha[mask]
|
||||
Phi = self.kernel(self.X, self.X)
|
||||
precision = np.diag(self.alpha) + self.beta * Phi.T @ Phi
|
||||
self.covariance = np.linalg.inv(precision)
|
||||
self.mean = self.beta * self.covariance @ Phi.T @ self.t
|
||||
|
||||
def predict(self, X, with_error=True):
|
||||
"""
|
||||
predict output with this model
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features)
|
||||
input
|
||||
with_error : bool
|
||||
if True, predict with standard deviation of the outputs
|
||||
|
||||
Returns
|
||||
-------
|
||||
mean : (sample_size,) ndarray
|
||||
mean of predictive distribution
|
||||
std : (sample_size,) ndarray
|
||||
standard deviation of predictive distribution
|
||||
"""
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
assert X.ndim == 2
|
||||
phi = self.kernel(X, self.X)
|
||||
mean = phi @ self.mean
|
||||
if with_error:
|
||||
var = 1 / self.beta + np.sum(phi @ self.covariance * phi, axis=1)
|
||||
return mean, np.sqrt(var)
|
||||
return mean
|
||||
@@ -0,0 +1,107 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
class SupportVectorClassifier(object):
|
||||
|
||||
def __init__(self, kernel, C=np.Inf):
|
||||
"""
|
||||
construct support vector classifier
|
||||
|
||||
Parameters
|
||||
----------
|
||||
kernel : Kernel
|
||||
kernel function to compute inner products
|
||||
C : float
|
||||
penalty of misclassification
|
||||
"""
|
||||
self.kernel = kernel
|
||||
self.C = C
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray, tol:float=1e-8):
|
||||
"""
|
||||
estimate support vectors and their parameters
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,) np.ndarray
|
||||
training dependent variable
|
||||
binary -1 or 1
|
||||
tol : float, optional
|
||||
numerical tolerance (the default is 1e-8)
|
||||
"""
|
||||
|
||||
N = len(t)
|
||||
coef = np.zeros(N)
|
||||
grad = np.ones(N)
|
||||
Gram = self.kernel(X, X)
|
||||
|
||||
while True:
|
||||
tg = t * grad
|
||||
mask_up = (t == 1) & (coef < self.C - tol)
|
||||
mask_up |= (t == -1) & (coef > tol)
|
||||
mask_down = (t == -1) & (coef < self.C - tol)
|
||||
mask_down |= (t == 1) & (coef > tol)
|
||||
i = np.where(mask_up)[0][np.argmax(tg[mask_up])]
|
||||
j = np.where(mask_down)[0][np.argmin(tg[mask_down])]
|
||||
if tg[i] < tg[j] + tol:
|
||||
self.b = 0.5 * (tg[i] + tg[j])
|
||||
break
|
||||
else:
|
||||
A = self.C - coef[i] if t[i] == 1 else coef[i]
|
||||
B = coef[j] if t[j] == 1 else self.C - coef[j]
|
||||
direction = (tg[i] - tg[j]) / (Gram[i, i] - 2 * Gram[i, j] + Gram[j, j])
|
||||
direction = min(A, B, direction)
|
||||
coef[i] += direction * t[i]
|
||||
coef[j] -= direction * t[j]
|
||||
grad -= direction * t * (Gram[i] - Gram[j])
|
||||
support_mask = coef > tol
|
||||
self.a = coef[support_mask]
|
||||
self.X = X[support_mask]
|
||||
self.t = t[support_mask]
|
||||
|
||||
def lagrangian_function(self):
|
||||
return (
|
||||
np.sum(self.a)
|
||||
- self.a
|
||||
@ (self.t * self.t[:, None] * self.kernel(self.X, self.X))
|
||||
@ self.a)
|
||||
|
||||
def predict(self, x):
|
||||
"""
|
||||
predict labels of the input
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x : (sample_size, n_features) ndarray
|
||||
input
|
||||
|
||||
Returns
|
||||
-------
|
||||
label : (sample_size,) ndarray
|
||||
predicted labels
|
||||
"""
|
||||
y = self.distance(x)
|
||||
label = np.sign(y)
|
||||
return label
|
||||
|
||||
def distance(self, x):
|
||||
"""
|
||||
calculate distance from the decision boundary
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x : (sample_size, n_features) ndarray
|
||||
input
|
||||
|
||||
Returns
|
||||
-------
|
||||
distance : (sample_size,) ndarray
|
||||
distance from the boundary
|
||||
"""
|
||||
distance = np.sum(
|
||||
self.a * self.t
|
||||
* self.kernel(x, self.X),
|
||||
axis=-1) + self.b
|
||||
return distance
|
||||
Reference in New Issue
Block a user