chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
+28
@@ -0,0 +1,28 @@
|
||||
from prml.linear.bayesian_logistic_regression import BayesianLogisticRegression
|
||||
from prml.linear.bayesian_regression import BayesianRegression
|
||||
from prml.linear.emprical_bayes_regression import EmpiricalBayesRegression
|
||||
from prml.linear.least_squares_classifier import LeastSquaresClassifier
|
||||
from prml.linear.linear_regression import LinearRegression
|
||||
from prml.linear.fishers_linear_discriminant import FishersLinearDiscriminant
|
||||
from prml.linear.logistic_regression import LogisticRegression
|
||||
from prml.linear.perceptron import Perceptron
|
||||
from prml.linear.ridge_regression import RidgeRegression
|
||||
from prml.linear.softmax_regression import SoftmaxRegression
|
||||
from prml.linear.variational_linear_regression import VariationalLinearRegression
|
||||
from prml.linear.variational_logistic_regression import VariationalLogisticRegression
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BayesianLogisticRegression",
|
||||
"BayesianRegression",
|
||||
"EmpiricalBayesRegression",
|
||||
"LeastSquaresClassifier",
|
||||
"LinearRegression",
|
||||
"FishersLinearDiscriminant",
|
||||
"LogisticRegression",
|
||||
"Perceptron",
|
||||
"RidgeRegression",
|
||||
"SoftmaxRegression",
|
||||
"VariationalLinearRegression",
|
||||
"VariationalLogisticRegression"
|
||||
]
|
||||
@@ -0,0 +1,66 @@
|
||||
import numpy as np
|
||||
from prml.linear.logistic_regression import LogisticRegression
|
||||
|
||||
|
||||
class BayesianLogisticRegression(LogisticRegression):
|
||||
"""
|
||||
Logistic regression model
|
||||
|
||||
w ~ Gaussian(0, alpha^(-1)I)
|
||||
y = sigmoid(X @ w)
|
||||
t ~ Bernoulli(t|y)
|
||||
"""
|
||||
|
||||
def __init__(self, alpha:float=1.):
|
||||
self.alpha = alpha
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100):
|
||||
"""
|
||||
bayesian estimation of logistic regression model
|
||||
using Laplace approximation
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training data independent variable
|
||||
t : (N,) np.ndarray
|
||||
training data dependent variable
|
||||
binary 0 or 1
|
||||
max_iter : int, optional
|
||||
maximum number of paramter update iteration (the default is 100)
|
||||
"""
|
||||
w = np.zeros(np.size(X, 1))
|
||||
eye = np.eye(np.size(X, 1))
|
||||
self.w_mean = np.copy(w)
|
||||
self.w_precision = self.alpha * eye
|
||||
for _ in range(max_iter):
|
||||
w_prev = np.copy(w)
|
||||
y = self._sigmoid(X @ w)
|
||||
grad = X.T @ (y - t) + self.w_precision @ (w - self.w_mean)
|
||||
hessian = (X.T * y * (1 - y)) @ X + self.w_precision
|
||||
try:
|
||||
w -= np.linalg.solve(hessian, grad)
|
||||
except np.linalg.LinAlgError:
|
||||
break
|
||||
if np.allclose(w, w_prev):
|
||||
break
|
||||
self.w_mean = w
|
||||
self.w_precision = hessian
|
||||
|
||||
def proba(self, X:np.ndarray):
|
||||
"""
|
||||
compute probability of input belonging class 1
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training data independent variable
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
probability of positive
|
||||
"""
|
||||
mu_a = X @ self.w_mean
|
||||
var_a = np.sum(np.linalg.solve(self.w_precision, X.T).T * X, axis=1)
|
||||
return self._sigmoid(mu_a / np.sqrt(1 + np.pi * var_a / 8))
|
||||
@@ -0,0 +1,87 @@
|
||||
import numpy as np
|
||||
from prml.linear.regression import Regression
|
||||
|
||||
|
||||
class BayesianRegression(Regression):
|
||||
"""
|
||||
Bayesian regression model
|
||||
|
||||
w ~ N(w|0, alpha^(-1)I)
|
||||
y = X @ w
|
||||
t ~ N(t|X @ w, beta^(-1))
|
||||
"""
|
||||
|
||||
def __init__(self, alpha:float=1., beta:float=1.):
|
||||
self.alpha = alpha
|
||||
self.beta = beta
|
||||
self.w_mean = None
|
||||
self.w_precision = None
|
||||
|
||||
def _is_prior_defined(self) -> bool:
|
||||
return self.w_mean is not None and self.w_precision is not None
|
||||
|
||||
def _get_prior(self, ndim:int) -> tuple:
|
||||
if self._is_prior_defined():
|
||||
return self.w_mean, self.w_precision
|
||||
else:
|
||||
return np.zeros(ndim), self.alpha * np.eye(ndim)
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray):
|
||||
"""
|
||||
bayesian update of parameters given training dataset
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, n_features) np.ndarray
|
||||
training data independent variable
|
||||
t : (N,) np.ndarray
|
||||
training data dependent variable
|
||||
"""
|
||||
|
||||
mean_prev, precision_prev = self._get_prior(np.size(X, 1))
|
||||
|
||||
w_precision = precision_prev + self.beta * X.T @ X
|
||||
w_mean = np.linalg.solve(
|
||||
w_precision,
|
||||
precision_prev @ mean_prev + self.beta * X.T @ t
|
||||
)
|
||||
self.w_mean = w_mean
|
||||
self.w_precision = w_precision
|
||||
self.w_cov = np.linalg.inv(self.w_precision)
|
||||
|
||||
def predict(self, X:np.ndarray, return_std:bool=False, sample_size:int=None):
|
||||
"""
|
||||
return mean (and standard deviation) of predictive distribution
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, n_features) np.ndarray
|
||||
independent variable
|
||||
return_std : bool, optional
|
||||
flag to return standard deviation (the default is False)
|
||||
sample_size : int, optional
|
||||
number of samples to draw from the predictive distribution
|
||||
(the default is None, no sampling from the distribution)
|
||||
|
||||
Returns
|
||||
-------
|
||||
y : (N,) np.ndarray
|
||||
mean of the predictive distribution
|
||||
y_std : (N,) np.ndarray
|
||||
standard deviation of the predictive distribution
|
||||
y_sample : (N, sample_size) np.ndarray
|
||||
samples from the predictive distribution
|
||||
"""
|
||||
|
||||
if sample_size is not None:
|
||||
w_sample = np.random.multivariate_normal(
|
||||
self.w_mean, self.w_cov, size=sample_size
|
||||
)
|
||||
y_sample = X @ w_sample.T
|
||||
return y_sample
|
||||
y = X @ self.w_mean
|
||||
if return_std:
|
||||
y_var = 1 / self.beta + np.sum(X @ self.w_cov * X, axis=1)
|
||||
y_std = np.sqrt(y_var)
|
||||
return y, y_std
|
||||
return y
|
||||
@@ -0,0 +1,5 @@
|
||||
class Classifier(object):
|
||||
"""
|
||||
Base class for classifiers
|
||||
"""
|
||||
pass
|
||||
@@ -0,0 +1,86 @@
|
||||
import numpy as np
|
||||
from prml.linear.bayesian_regression import BayesianRegression
|
||||
|
||||
|
||||
class EmpiricalBayesRegression(BayesianRegression):
|
||||
"""
|
||||
Empirical Bayes Regression model
|
||||
a.k.a.
|
||||
type 2 maximum likelihood,
|
||||
generalized maximum likelihood,
|
||||
evidence approximation
|
||||
|
||||
w ~ N(w|0, alpha^(-1)I)
|
||||
y = X @ w
|
||||
t ~ N(t|X @ w, beta^(-1))
|
||||
evidence function p(t|X,alpha,beta) = S p(t|w;X,beta)p(w|0;alpha) dw
|
||||
"""
|
||||
|
||||
def __init__(self, alpha:float=1., beta:float=1.):
|
||||
super().__init__(alpha, beta)
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100):
|
||||
"""
|
||||
maximization of evidence function with respect to
|
||||
the hyperparameters alpha and beta given training dataset
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,) np.ndarray
|
||||
training dependent variable
|
||||
max_iter : int
|
||||
maximum number of iteration
|
||||
"""
|
||||
M = X.T @ X
|
||||
eigenvalues = np.linalg.eigvalsh(M)
|
||||
eye = np.eye(np.size(X, 1))
|
||||
N = len(t)
|
||||
for _ in range(max_iter):
|
||||
params = [self.alpha, self.beta]
|
||||
|
||||
w_precision = self.alpha * eye + self.beta * X.T @ X
|
||||
w_mean = self.beta * np.linalg.solve(w_precision, X.T @ t)
|
||||
|
||||
gamma = np.sum(eigenvalues / (self.alpha + eigenvalues))
|
||||
self.alpha = float(gamma / np.sum(w_mean ** 2).clip(min=1e-10))
|
||||
self.beta = float(
|
||||
(N - gamma) / np.sum(np.square(t - X @ w_mean))
|
||||
)
|
||||
if np.allclose(params, [self.alpha, self.beta]):
|
||||
break
|
||||
self.w_mean = w_mean
|
||||
self.w_precision = w_precision
|
||||
self.w_cov = np.linalg.inv(w_precision)
|
||||
|
||||
def _log_prior(self, w):
|
||||
return -0.5 * self.alpha * np.sum(w ** 2)
|
||||
|
||||
def _log_likelihood(self, X, t, w):
|
||||
return -0.5 * self.beta * np.square(t - X @ w).sum()
|
||||
|
||||
def _log_posterior(self, X, t, w):
|
||||
return self._log_likelihood(X, t, w) + self._log_prior(w)
|
||||
|
||||
def log_evidence(self, X:np.ndarray, t:np.ndarray):
|
||||
"""
|
||||
logarithm or the evidence function
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
indenpendent variable
|
||||
t : (N,) np.ndarray
|
||||
dependent variable
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
log evidence
|
||||
"""
|
||||
N = len(t)
|
||||
D = np.size(X, 1)
|
||||
return 0.5 * (
|
||||
D * np.log(self.alpha) + N * np.log(self.beta)
|
||||
- np.linalg.slogdet(self.w_precision)[1] - D * np.log(2 * np.pi)
|
||||
) + self._log_posterior(X, t, self.w_mean)
|
||||
@@ -0,0 +1,80 @@
|
||||
import numpy as np
|
||||
from prml.linear.classifier import Classifier
|
||||
from prml.rv.gaussian import Gaussian
|
||||
|
||||
|
||||
class FishersLinearDiscriminant(Classifier):
|
||||
"""
|
||||
Fisher's Linear discriminant model
|
||||
"""
|
||||
|
||||
def __init__(self, w:np.ndarray=None, threshold:float=None):
|
||||
self.w = w
|
||||
self.threshold = threshold
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray):
|
||||
"""
|
||||
estimate parameter given training dataset
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training dataset independent variable
|
||||
t : (N,) np.ndarray
|
||||
training dataset dependent variable
|
||||
binary 0 or 1
|
||||
"""
|
||||
X0 = X[t == 0]
|
||||
X1 = X[t == 1]
|
||||
m0 = np.mean(X0, axis=0)
|
||||
m1 = np.mean(X1, axis=0)
|
||||
cov_inclass = np.cov(X0, rowvar=False) + np.cov(X1, rowvar=False)
|
||||
self.w = np.linalg.solve(cov_inclass, m1 - m0)
|
||||
self.w /= np.linalg.norm(self.w).clip(min=1e-10)
|
||||
|
||||
g0 = Gaussian()
|
||||
g0.fit((X0 @ self.w))
|
||||
g1 = Gaussian()
|
||||
g1.fit((X1 @ self.w))
|
||||
root = np.roots([
|
||||
g1.var - g0.var,
|
||||
2 * (g0.var * g1.mu - g1.var * g0.mu),
|
||||
g1.var * g0.mu ** 2 - g0.var * g1.mu ** 2
|
||||
- g1.var * g0.var * np.log(g1.var / g0.var)
|
||||
])
|
||||
if g0.mu < root[0] < g1.mu or g1.mu < root[0] < g0.mu:
|
||||
self.threshold = root[0]
|
||||
else:
|
||||
self.threshold = root[1]
|
||||
|
||||
def transform(self, X:np.ndarray):
|
||||
"""
|
||||
project data
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable
|
||||
|
||||
Returns
|
||||
-------
|
||||
y : (N,) np.ndarray
|
||||
projected data
|
||||
"""
|
||||
return X @ self.w
|
||||
|
||||
def classify(self, X:np.ndarray):
|
||||
"""
|
||||
classify input data
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable to be classified
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
binary class for each input
|
||||
"""
|
||||
return (X @ self.w > self.threshold).astype(np.int)
|
||||
@@ -0,0 +1,48 @@
|
||||
import numpy as np
|
||||
from prml.linear.classifier import Classifier
|
||||
from prml.preprocess.label_transformer import LabelTransformer
|
||||
|
||||
|
||||
class LeastSquaresClassifier(Classifier):
|
||||
"""
|
||||
Least squares classifier model
|
||||
|
||||
X : (N, D)
|
||||
W : (D, K)
|
||||
y = argmax_k X @ W
|
||||
"""
|
||||
|
||||
def __init__(self, W:np.ndarray=None):
|
||||
self.W = W
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray):
|
||||
"""
|
||||
least squares fitting for classification
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,) or (N, K) np.ndarray
|
||||
training dependent variable
|
||||
in class index (N,) or one-of-k coding (N,K)
|
||||
"""
|
||||
if t.ndim == 1:
|
||||
t = LabelTransformer().encode(t)
|
||||
self.W = np.linalg.pinv(X) @ t
|
||||
|
||||
def classify(self, X:np.ndarray):
|
||||
"""
|
||||
classify input data
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable to be classified
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
class index for each input
|
||||
"""
|
||||
return np.argmax(X @ self.W, axis=-1)
|
||||
@@ -0,0 +1,48 @@
|
||||
import numpy as np
|
||||
from prml.linear.regression import Regression
|
||||
|
||||
|
||||
class LinearRegression(Regression):
|
||||
"""
|
||||
Linear regression model
|
||||
y = X @ w
|
||||
t ~ N(t|X @ w, var)
|
||||
"""
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray):
|
||||
"""
|
||||
perform least squares fitting
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,) np.ndarray
|
||||
training dependent variable
|
||||
"""
|
||||
self.w = np.linalg.pinv(X) @ t
|
||||
self.var = np.mean(np.square(X @ self.w - t))
|
||||
|
||||
def predict(self, X:np.ndarray, return_std:bool=False):
|
||||
"""
|
||||
make prediction given input
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
samples to predict their output
|
||||
return_std : bool, optional
|
||||
returns standard deviation of each predition if True
|
||||
|
||||
Returns
|
||||
-------
|
||||
y : (N,) np.ndarray
|
||||
prediction of each sample
|
||||
y_std : (N,) np.ndarray
|
||||
standard deviation of each predition
|
||||
"""
|
||||
y = X @ self.w
|
||||
if return_std:
|
||||
y_std = np.sqrt(self.var) + np.zeros_like(y)
|
||||
return y, y_std
|
||||
return y
|
||||
@@ -0,0 +1,77 @@
|
||||
import numpy as np
|
||||
from prml.linear.classifier import Classifier
|
||||
|
||||
|
||||
class LogisticRegression(Classifier):
|
||||
"""
|
||||
Logistic regression model
|
||||
|
||||
y = sigmoid(X @ w)
|
||||
t ~ Bernoulli(t|y)
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _sigmoid(a):
|
||||
return np.tanh(a * 0.5) * 0.5 + 0.5
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100):
|
||||
"""
|
||||
maximum likelihood estimation of logistic regression model
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training data independent variable
|
||||
t : (N,) np.ndarray
|
||||
training data dependent variable
|
||||
binary 0 or 1
|
||||
max_iter : int, optional
|
||||
maximum number of paramter update iteration (the default is 100)
|
||||
"""
|
||||
w = np.zeros(np.size(X, 1))
|
||||
for _ in range(max_iter):
|
||||
w_prev = np.copy(w)
|
||||
y = self._sigmoid(X @ w)
|
||||
grad = X.T @ (y - t)
|
||||
hessian = (X.T * y * (1 - y)) @ X
|
||||
try:
|
||||
w -= np.linalg.solve(hessian, grad)
|
||||
except np.linalg.LinAlgError:
|
||||
break
|
||||
if np.allclose(w, w_prev):
|
||||
break
|
||||
self.w = w
|
||||
|
||||
def proba(self, X:np.ndarray):
|
||||
"""
|
||||
compute probability of input belonging class 1
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training data independent variable
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
probability of positive
|
||||
"""
|
||||
return self._sigmoid(X @ self.w)
|
||||
|
||||
def classify(self, X:np.ndarray, threshold:float=0.5):
|
||||
"""
|
||||
classify input data
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable to be classified
|
||||
threshold : float, optional
|
||||
threshold of binary classification (default is 0.5)
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
binary class for each input
|
||||
"""
|
||||
return (self.proba(X) > threshold).astype(np.int)
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
import numpy as np
|
||||
from prml.linear.classifier import Classifier
|
||||
|
||||
|
||||
class Perceptron(Classifier):
|
||||
"""
|
||||
Perceptron model
|
||||
"""
|
||||
|
||||
def fit(self, X, t, max_epoch=100):
|
||||
"""
|
||||
fit perceptron model on given input pair
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,)
|
||||
training dependent variable
|
||||
binary -1 or 1
|
||||
max_epoch : int, optional
|
||||
maximum number of epoch (the default is 100)
|
||||
"""
|
||||
self.w = np.zeros(np.size(X, 1))
|
||||
for _ in range(max_epoch):
|
||||
N = len(t)
|
||||
index = np.random.permutation(N)
|
||||
X = X[index]
|
||||
t = t[index]
|
||||
for x, label in zip(X, t):
|
||||
self.w += x * label
|
||||
if (X @ self.w * t > 0).all():
|
||||
break
|
||||
else:
|
||||
continue
|
||||
break
|
||||
|
||||
def classify(self, X):
|
||||
"""
|
||||
classify input data
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable to be classified
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
binary class (-1 or 1) for each input
|
||||
"""
|
||||
return np.sign(X @ self.w).astype(np.int)
|
||||
@@ -0,0 +1,5 @@
|
||||
class Regression(object):
|
||||
"""
|
||||
Base class for regressors
|
||||
"""
|
||||
pass
|
||||
@@ -0,0 +1,44 @@
|
||||
import numpy as np
|
||||
from prml.linear.regression import Regression
|
||||
|
||||
|
||||
class RidgeRegression(Regression):
|
||||
"""
|
||||
Ridge regression model
|
||||
|
||||
w* = argmin |t - X @ w| + alpha * |w|_2^2
|
||||
"""
|
||||
|
||||
def __init__(self, alpha:float=1.):
|
||||
self.alpha = alpha
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray):
|
||||
"""
|
||||
maximum a posteriori estimation of parameter
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training data independent variable
|
||||
t : (N,) np.ndarray
|
||||
training data dependent variable
|
||||
"""
|
||||
|
||||
eye = np.eye(np.size(X, 1))
|
||||
self.w = np.linalg.solve(self.alpha * eye + X.T @ X, X.T @ t)
|
||||
|
||||
def predict(self, X:np.ndarray):
|
||||
"""
|
||||
make prediction given input
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
samples to predict their output
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
prediction of each input
|
||||
"""
|
||||
return X @ self.w
|
||||
@@ -0,0 +1,83 @@
|
||||
import numpy as np
|
||||
from prml.linear.classifier import Classifier
|
||||
from prml.preprocess.label_transformer import LabelTransformer
|
||||
|
||||
|
||||
class SoftmaxRegression(Classifier):
|
||||
"""
|
||||
Softmax regression model
|
||||
aka
|
||||
multinomial logistic regression,
|
||||
multiclass logistic regression,
|
||||
maximum entropy classifier.
|
||||
|
||||
y = softmax(X @ W)
|
||||
t ~ Categorical(t|y)
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _softmax(a):
|
||||
a_max = np.max(a, axis=-1, keepdims=True)
|
||||
exp_a = np.exp(a - a_max)
|
||||
return exp_a / np.sum(exp_a, axis=-1, keepdims=True)
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100, learning_rate:float=0.1):
|
||||
"""
|
||||
maximum likelihood estimation of the parameter
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,) or (N, K) np.ndarray
|
||||
training dependent variable
|
||||
in class index or one-of-k encoding
|
||||
max_iter : int, optional
|
||||
maximum number of iteration (the default is 100)
|
||||
learning_rate : float, optional
|
||||
learning rate of gradient descent (the default is 0.1)
|
||||
"""
|
||||
if t.ndim == 1:
|
||||
t = LabelTransformer().encode(t)
|
||||
self.n_classes = np.size(t, 1)
|
||||
W = np.zeros((np.size(X, 1), self.n_classes))
|
||||
for _ in range(max_iter):
|
||||
W_prev = np.copy(W)
|
||||
y = self._softmax(X @ W)
|
||||
grad = X.T @ (y - t)
|
||||
W -= learning_rate * grad
|
||||
if np.allclose(W, W_prev):
|
||||
break
|
||||
self.W = W
|
||||
|
||||
def proba(self, X:np.ndarray):
|
||||
"""
|
||||
compute probability of input belonging each class
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N, K) np.ndarray
|
||||
probability of each class
|
||||
"""
|
||||
return self._softmax(X @ self.W)
|
||||
|
||||
def classify(self, X:np.ndarray):
|
||||
"""
|
||||
classify input data
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable to be classified
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
class index for each input
|
||||
"""
|
||||
return np.argmax(self.proba(X), axis=-1)
|
||||
@@ -0,0 +1,93 @@
|
||||
import numpy as np
|
||||
from prml.linear.regression import Regression
|
||||
|
||||
|
||||
class VariationalLinearRegression(Regression):
|
||||
"""
|
||||
variational bayesian estimation of linear regression model
|
||||
p(w,alpha|X,t)
|
||||
~ q(w)q(alpha)
|
||||
= N(w|w_mean, w_var)Gamma(alpha|a,b)
|
||||
|
||||
Attributes
|
||||
----------
|
||||
a : float
|
||||
a parameter of variational posterior gamma distribution
|
||||
b : float
|
||||
another parameter of variational posterior gamma distribution
|
||||
w_mean : (n_features,) ndarray
|
||||
mean of variational posterior gaussian distribution
|
||||
w_var : (n_features, n_feautures) ndarray
|
||||
variance of variational posterior gaussian distribution
|
||||
n_iter : int
|
||||
number of iterations performed
|
||||
"""
|
||||
|
||||
def __init__(self, beta:float=1., a0:float=1., b0:float=1.):
|
||||
"""
|
||||
construct variational linear regressor
|
||||
Parameters
|
||||
----------
|
||||
beta : float
|
||||
precision of observation noise
|
||||
a0 : float
|
||||
a parameter of prior gamma distribution
|
||||
Gamma(alpha|a0,b0)
|
||||
b0 : float
|
||||
another parameter of prior gamma distribution
|
||||
Gamma(alpha|a0,b0)
|
||||
"""
|
||||
self.beta = beta
|
||||
self.a0 = a0
|
||||
self.b0 = b0
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray, iter_max:int=100):
|
||||
"""
|
||||
variational bayesian estimation of parameter
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,) np.ndarray
|
||||
training dependent variable
|
||||
iter_max : int, optional
|
||||
maximum number of iteration (the default is 100)
|
||||
"""
|
||||
D = np.size(X, 1)
|
||||
self.a = self.a0 + 0.5 * D
|
||||
self.b = self.b0
|
||||
I = np.eye(D)
|
||||
for _ in range(iter_max):
|
||||
param = self.b
|
||||
self.w_var = np.linalg.inv(self.a * I / self.b + self.beta * X.T @ X)
|
||||
self.w_mean = self.beta * self.w_var @ X.T @ t
|
||||
self.b = self.b0 + 0.5 * (np.sum(self.w_mean ** 2) + np.trace(self.w_var))
|
||||
if np.allclose(self.b, param):
|
||||
break
|
||||
|
||||
def predict(self, X:np.ndarray, return_std:bool=False):
|
||||
"""
|
||||
make prediction of input
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable
|
||||
return_std : bool, optional
|
||||
return standard deviation of predictive distribution if True
|
||||
(the default is False)
|
||||
|
||||
Returns
|
||||
-------
|
||||
y : (N,) np.ndarray
|
||||
mean of predictive distribution
|
||||
y_std : (N,) np.ndarray
|
||||
standard deviation of predictive distribution
|
||||
"""
|
||||
y = X @ self.w_mean
|
||||
if return_std:
|
||||
y_var = 1 / self.beta + np.sum(X @ self.w_var * X, axis=1)
|
||||
y_std = np.sqrt(y_var)
|
||||
return y, y_std
|
||||
return y
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
import numpy as np
|
||||
from prml.linear.logistic_regression import LogisticRegression
|
||||
|
||||
|
||||
class VariationalLogisticRegression(LogisticRegression):
|
||||
|
||||
def __init__(self, alpha:float=None, a0:float=1., b0:float=1.):
|
||||
"""
|
||||
construct variational logistic regressor
|
||||
|
||||
Parameters
|
||||
----------
|
||||
alpha : float
|
||||
precision parameter of the prior
|
||||
if None, this is also the subject to estimate
|
||||
a0 : float
|
||||
a parameter of hyper prior Gamma dist.
|
||||
Gamma(alpha|a0,b0)
|
||||
if alpha is not None, this argument will be ignored
|
||||
b0 : float
|
||||
another parameter of hyper prior Gamma dist.
|
||||
Gamma(alpha|a0,b0)
|
||||
if alpha is not None, this argument will be ignored
|
||||
"""
|
||||
if alpha is not None:
|
||||
self.__alpha = alpha
|
||||
else:
|
||||
self.a0 = a0
|
||||
self.b0 = b0
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray, iter_max:int=1000):
|
||||
"""
|
||||
variational bayesian estimation of the parameter
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,) np.ndarray
|
||||
training dependent variable
|
||||
iter_max : int, optional
|
||||
maximum number of iteration (the default is 1000)
|
||||
"""
|
||||
N, D = X.shape
|
||||
if hasattr(self, "a0"):
|
||||
self.a = self.a0 + 0.5 * D
|
||||
xi = np.random.uniform(-1, 1, size=N)
|
||||
I = np.eye(D)
|
||||
param = np.copy(xi)
|
||||
for _ in range(iter_max):
|
||||
lambda_ = np.tanh(xi) * 0.25 / xi
|
||||
self.w_var = np.linalg.inv(I / self.alpha + 2 * (lambda_ * X.T) @ X)
|
||||
self.w_mean = self.w_var @ np.sum(X.T * (t - 0.5), axis=1)
|
||||
xi = np.sqrt(np.sum(X @ (self.w_var + self.w_mean * self.w_mean[:, None]) * X, axis=-1))
|
||||
if np.allclose(xi, param):
|
||||
break
|
||||
else:
|
||||
param = np.copy(xi)
|
||||
|
||||
@property
|
||||
def alpha(self):
|
||||
if hasattr(self, "__alpha"):
|
||||
return self.__alpha
|
||||
else:
|
||||
try:
|
||||
self.b = self.b0 + 0.5 * (np.sum(self.w_mean ** 2) + np.trace(self.w_var))
|
||||
except AttributeError:
|
||||
self.b = self.b0
|
||||
return self.a / self.b
|
||||
|
||||
def proba(self, X:np.ndarray):
|
||||
"""
|
||||
compute probability of input belonging class 1
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training data independent variable
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
probability of positive
|
||||
"""
|
||||
mu_a = X @ self.w_mean
|
||||
var_a = np.sum(X @ self.w_var * X, axis=1)
|
||||
y = self._sigmoid(mu_a / np.sqrt(1 + np.pi * var_a / 8))
|
||||
return y
|
||||
Reference in New Issue
Block a user