chore: import upstream snapshot with attribution

books/PRML/PRML-master-Python/prml/__init__.py

@@ -0,0 +1,24 @@
+from prml import (
+    bayesnet,
+    clustering,
+    dimreduction,
+    kernel,
+    linear,
+    markov,
+    nn,
+    rv,
+    sampling
+)
+
+
+__all__ = [
+    "bayesnet",
+    "clustering",
+    "dimreduction",
+    "kernel",
+    "linear",
+    "markov",
+    "nn",
+    "rv",
+    "sampling"
+]

books/PRML/PRML-master-Python/prml/bayesnet/__init__.py

@@ -0,0 +1,7 @@
+from prml.bayesnet.discrete import discrete, DiscreteVariable
+
+
+__all__ = [
+    "DiscreteVariable",
+    "discrete"
+]

books/PRML/PRML-master-Python/prml/bayesnet/discrete.py

@@ -0,0 +1,242 @@
+import numpy as np
+from prml.bayesnet.probability_function import ProbabilityFunction
+from prml.bayesnet.random_variable import RandomVariable
+
+
+class DiscreteVariable(RandomVariable):
+    """
+    Discrete random variable
+    """
+
+    def __init__(self, n_class:int):
+        """
+        intialize a discrete random variable
+
+        parameters
+        ----------
+        n_class : int
+            number of classes
+
+        Attributes
+        ----------
+        parent : DiscreteProbability, optional
+            parent node this variable came out from
+        message_from : dict
+            dictionary of message from neighbor node and itself
+        child : list of DiscreteProbability
+            probability function this variable is conditioning
+        proba : np.ndarray
+            current estimate
+        """
+        self.n_class = n_class
+        self.parent = []
+        self.message_from = {self: np.ones(n_class)}
+        self.child = []
+        self.is_observed = False
+
+    def __repr__(self):
+        string = f"DiscreteVariable("
+        if self.is_observed:
+            string += f"observed={self.proba})"
+        else:
+            string += f"proba={self.proba})"
+        return string
+
+    def add_parent(self, parent):
+        self.parent.append(parent)
+
+    def add_child(self, child):
+        self.child.append(child)
+        self.message_from[child] = np.ones(self.n_class)
+
+    @property
+    def proba(self):
+        return self.posterior
+
+    def receive_message(self, message, giver, proprange):
+        self.message_from[giver] = message
+        self.summarize_message()
+        self.send_message(proprange, exclude=giver)
+
+    def summarize_message(self):
+        if self.is_observed:
+            self.prior = self.message_from[self]
+            self.likelihood = self.prior
+            self.posterior = self.prior
+            return
+
+        self.prior = np.ones(self.n_class)
+        for func in self.parent:
+            self.prior *= self.message_from[func]
+        self.prior /= np.sum(self.prior, keepdims=True)
+
+        self.likelihood = np.copy(self.message_from[self])
+        for func in self.child:
+            self.likelihood *= self.message_from[func]
+
+        self.posterior = self.prior * self.likelihood
+        self.posterior /= self.posterior.sum()
+
+    def send_message(self, proprange=-1, exclude=None):
+        for func in self.parent:
+            if func is not exclude:
+                func.receive_message(self.likelihood, self, proprange)
+        for func in self.child:
+            if func is not exclude:
+                func.receive_message(self.prior, self, proprange)
+
+    def observe(self, data:int, proprange=-1):
+        """
+        set observed data of this variable
+
+        Parameters
+        ----------
+        data : int
+            observed data of this variable
+            This must be smaller than n_class and must be non-negative
+        propagate : int, optional
+            Range to propagate the observation effect to the other random variable using belief propagation alg.
+            If proprange=1, the effect only propagate to the neighboring random variables.
+            Default is -1, which is infinite range.
+        """
+        assert(0 <= data < self.n_class)
+        self.is_observed = True
+        self.receive_message(np.eye(self.n_class)[data], self, proprange=proprange)
+
+
+class DiscreteProbability(ProbabilityFunction):
+    """
+    Discrete probability function
+    """
+
+    def __init__(self, table, *condition, out=None, name=None):
+        """
+        initialize discrete probability function
+
+        Parameters
+        ----------
+        table : (K, ...) np.ndarray or array-like
+            probability table
+            If a discrete variable A is conditioned with B and C,
+            table[a,b,c] give probability of A=a when B=b and C=c.
+            Thus, the sum along the first axis should equal to 1.
+            If a table is 1 dimensional, the variable is not conditioned.
+        condition : tuple of DiscreteVariable, optional
+            parent node, discrete variable this function is conidtioned by
+            len(condition) should equal to (table.ndim - 1)
+            (Default is (), which means no condition)
+        out : DiscreteVariable or list of DiscreteVariable, optional
+            output of this discrete probability function
+            Default is None which construct a new output instance
+        name : str
+            name of this discrete probability function
+        """
+        self.table = np.asarray(table)
+        self.condition = condition
+        if condition:
+            for var in condition:
+                var.add_child(self)
+        self.message_from = {var: var.prior for var in condition}
+
+        if out is None:
+            self.out = [DiscreteVariable(len(table))]
+        elif isinstance(out, DiscreteVariable):
+            self.out = [out]
+        else:
+            self.out = out
+
+        for i, random_variable in enumerate(self.out):
+            random_variable.add_parent(self)
+            self.message_from[random_variable] = np.ones(np.size(self.table, i))
+
+        for random_variable in self.out:
+            self.send_message_to(random_variable, proprange=0)
+
+        self.name = name
+
+    def __repr__(self):
+        if self.name is not None:
+            return self.name
+        else:
+            return super().__repr__()
+
+    def receive_message(self, message, giver, proprange):
+        self.message_from[giver] = message
+        if proprange:
+            self.send_message(proprange, exclude=giver)
+
+    @staticmethod
+    def expand_dims(x, ndim, axis):
+        shape = [-1 if i == axis else 1 for i in range(ndim)]
+        return x.reshape(*shape)
+
+    def compute_message_to(self, destination):
+        proba = np.copy(self.table)
+        for i, random_variable in enumerate(self.out):
+            if random_variable is destination:
+                index = i
+                continue
+            message = self.message_from[random_variable]
+            proba *= self.expand_dims(message, proba.ndim, i)
+        for i, random_variable in enumerate(self.condition, len(self.out)):
+            if random_variable is destination:
+                index = i
+                continue
+            message = self.message_from[random_variable]
+            proba *= self.expand_dims(message, proba.ndim, i)
+        axis = list(range(proba.ndim))
+        axis.remove(index)
+        message = np.sum(proba, axis=tuple(axis))
+        message /= np.sum(message, keepdims=True)
+        return message
+
+    def send_message_to(self, destination, proprange=-1):
+        message = self.compute_message_to(destination)
+        destination.receive_message(message, self, proprange)
+
+    def send_message(self, proprange, exclude=None):
+        proprange = proprange - 1
+
+        for random_variable in self.out:
+            if random_variable is not exclude:
+                self.send_message_to(random_variable, proprange)
+
+        if proprange == 0: return
+
+        for random_variable in self.condition:
+            if random_variable is not exclude:
+                self.send_message_to(random_variable, proprange - 1)
+
+
+def discrete(table, *condition, out=None, name=None):
+    """
+    discrete probability function
+
+    Parameters
+    ----------
+    table : (K, ...) np.ndarray or array-like
+        probability table
+        If a discrete variable A is conditioned with B and C,
+        table[a,b,c] give probability of A=a when B=b and C=c.
+        Thus, the sum along the first axis should equal to 1.
+        If a table is 1 dimensional, the variable is not conditioned.
+    condition : tuple of DiscreteVariable, optional
+        parent node, discrete variable this function is conidtioned by
+        len(condition) should equal to (table.ndim - 1)
+        (Default is (), which means no condition)
+    out : DiscreteVariable, optional
+        output of this discrete probability function
+        Default is None which construct a new output instance
+    name : str
+        name of the discrete probability function
+
+    Returns
+    -------
+    DiscreteVariable
+        output discrete random variable of discrete probability function
+    """
+    function = DiscreteProbability(table, *condition, out=out, name=name)
+    if len(function.out) == 1:
+        return function.out[0]
+    else:
+        return function.out

books/PRML/PRML-master-Python/prml/bayesnet/probability_function.py

@@ -0,0 +1,2 @@
+class ProbabilityFunction(object):
+    pass

books/PRML/PRML-master-Python/prml/bayesnet/random_variable.py

@@ -0,0 +1,4 @@
+class RandomVariable(object):
+    """
+    Base class for random variable
+    """

books/PRML/PRML-master-Python/prml/clustering/__init__.py

@@ -0,0 +1,6 @@
+from .k_means import KMeans
+
+
+__all__ = [
+    "KMeans"
+]

books/PRML/PRML-master-Python/prml/clustering/k_means.py

@@ -0,0 +1,53 @@
+import numpy as np
+from scipy.spatial.distance import cdist
+
+
+class KMeans(object):
+
+    def __init__(self, n_clusters):
+        self.n_clusters = n_clusters
+
+    def fit(self, X, iter_max=100):
+        """
+        perform k-means algorithm
+
+        Parameters
+        ----------
+        X : (sample_size, n_features) ndarray
+            input data
+        iter_max : int
+            maximum number of iterations
+
+        Returns
+        -------
+        centers : (n_clusters, n_features) ndarray
+            center of each cluster
+        """
+        I = np.eye(self.n_clusters)
+        centers = X[np.random.choice(len(X), self.n_clusters, replace=False)]
+        for _ in range(iter_max):
+            prev_centers = np.copy(centers)
+            D = cdist(X, centers)
+            cluster_index = np.argmin(D, axis=1)
+            cluster_index = I[cluster_index]
+            centers = np.sum(X[:, None, :] * cluster_index[:, :, None], axis=0) / np.sum(cluster_index, axis=0)[:, None]
+            if np.allclose(prev_centers, centers):
+                break
+        self.centers = centers
+
+    def predict(self, X):
+        """
+        calculate closest cluster center index
+
+        Parameters
+        ----------
+        X : (sample_size, n_features) ndarray
+            input data
+
+        Returns
+        -------
+        index : (sample_size,) ndarray
+            indicates which cluster they belong
+        """
+        D = cdist(X, self.centers)
+        return np.argmin(D, axis=1)

books/PRML/PRML-master-Python/prml/dimreduction/__init__.py

@@ -0,0 +1,10 @@
+from prml.dimreduction.autoencoder import Autoencoder
+from prml.dimreduction.bayesian_pca import BayesianPCA
+from prml.dimreduction.pca import PCA
+
+
+__all__ = [
+    "Autoencoder",
+    "BayesianPCA",
+    "PCA",
+]

books/PRML/PRML-master-Python/prml/dimreduction/autoencoder.py

@@ -0,0 +1,38 @@
+import numpy as np
+from prml import nn
+
+
+class Autoencoder(nn.Network):
+
+    def __init__(self, *args):
+        self.n_unit = len(args)
+        super().__init__()
+        for i in range(self.n_unit - 1):
+            self.parameter[f"w_encode{i}"] = nn.Parameter(np.random.randn(args[i], args[i + 1]))
+            self.parameter[f"b_encode{i}"] = nn.Parameter(np.zeros(args[i + 1]))
+            self.parameter[f"w_decode{i}"] = nn.Parameter(np.random.randn(args[i + 1], args[i]))
+            self.parameter[f"b_decode{i}"] = nn.Parameter(np.zeros(args[i]))
+
+    def transform(self, x):
+        h = x
+        for i in range(self.n_unit - 1):
+            h = nn.tanh(h @ self.parameter[f"w_encode{i}"] + self.parameter[f"b_encode{i}"])
+        return h.value
+
+    def forward(self, x):
+        h = x
+        for i in range(self.n_unit - 1):
+            h = nn.tanh(h @ self.parameter[f"w_encode{i}"] + self.parameter[f"b_encode{i}"])
+        for i in range(self.n_unit - 2, 0, -1):
+            h = nn.tanh(h @ self.parameter[f"w_decode{i}"] + self.parameter[f"b_decode{i}"])
+        x_ = h @ self.parameter["w_decode0"] + self.parameter["b_decode0"]
+        self.px = nn.random.Gaussian(x_, 1., data=x)
+
+    def fit(self, x, n_iter=100, learning_rate=1e-3):
+        optimizer = nn.optimizer.Adam(self.parameter, learning_rate)
+        for _ in range(n_iter):
+            self.clear()
+            self.forward(x)
+            log_likelihood = self.log_pdf()
+            log_likelihood.backward()
+            optimizer.update()

books/PRML/PRML-master-Python/prml/dimreduction/bayesian_pca.py

@@ -0,0 +1,59 @@
+import numpy as np
+from prml.dimreduction.pca import PCA
+
+
+class BayesianPCA(PCA):
+
+    def fit(self, X, iter_max=100, initial="random"):
+        """
+        empirical bayes estimation of pca parameters
+
+        Parameters
+        ----------
+        X : (sample_size, n_features) ndarray
+            input data
+        iter_max : int
+            maximum number of em steps
+
+        Returns
+        -------
+        mean : (n_features,) ndarray
+            sample mean fo the input data
+        W : (n_features, n_components) ndarray
+            projection matrix
+        var : float
+            variance of observation noise
+        """
+        initial_list = ["random", "eigen"]
+        self.mean = np.mean(X, axis=0)
+        self.I = np.eye(self.n_components)
+        if initial not in initial_list:
+            print("availabel initializations are {}".format(initial_list))
+        if initial == "random":
+            self.W = np.eye(np.size(X, 1), self.n_components)
+            self.var = 1.
+        elif initial == "eigen":
+            self.eigen(X)
+        self.alpha = len(self.mean) / np.sum(self.W ** 2, axis=0).clip(min=1e-10)
+        for i in range(iter_max):
+            W = np.copy(self.W)
+            stats = self._expectation(X - self.mean)
+            self._maximization(X - self.mean, *stats)
+            self.alpha = len(self.mean) / np.sum(self.W ** 2, axis=0).clip(min=1e-10)
+            if np.allclose(W, self.W):
+                break
+        self.n_iter = i + 1
+
+    def _maximization(self, X, Ez, Ezz):
+        self.W = X.T @ Ez @ np.linalg.inv(np.sum(Ezz, axis=0) + self.var * np.diag(self.alpha))
+        self.var = np.mean(
+            np.mean(X ** 2, axis=-1)
+            - 2 * np.mean(Ez @ self.W.T * X, axis=-1)
+            + np.trace((Ezz @ self.W.T @ self.W).T) / len(self.mean))
+
+    def maximize(self, D, Ez, Ezz):
+        self.W = D.T.dot(Ez).dot(np.linalg.inv(np.sum(Ezz, axis=0) + self.var * np.diag(self.alpha)))
+        self.var = np.mean(
+            np.mean(D ** 2, axis=-1)
+            - 2 * np.mean(Ez.dot(self.W.T) * D, axis=-1)
+            + np.trace(Ezz.dot(self.W.T).dot(self.W).T) / self.ndim)

books/PRML/PRML-master-Python/prml/dimreduction/pca.py

@@ -0,0 +1,156 @@
+import numpy as np
+
+
+class PCA(object):
+
+    def __init__(self, n_components):
+        """
+        construct principal component analysis
+
+        Parameters
+        ----------
+        n_components : int
+            number of components
+        """
+        assert isinstance(n_components, int)
+        self.n_components = n_components
+
+    def fit(self, X, method="eigen", iter_max=100):
+        """
+        maximum likelihood estimate of pca parameters
+        x ~ \int_z N(x|Wz+mu,sigma^2)N(z|0,I)dz
+
+        Parameters
+        ----------
+        X : (sample_size, n_features) ndarray
+            input data
+        method : str
+            method to estimate the parameters
+            ["eigen", "em"]
+        iter_max : int
+            maximum number of iterations for em algorithm
+
+        Attributes
+        ----------
+        mean : (n_features,) ndarray
+            sample mean of the data
+        W : (n_features, n_components) ndarray
+            projection matrix
+        var : float
+            variance of observation noise
+        C : (n_features, n_features) ndarray
+            variance of the marginal dist N(x|mean,C)
+        Cinv : (n_features, n_features) ndarray
+            precision of the marginal dist N(x|mean, C)
+        """
+        method_list = ["eigen", "em"]
+        if method not in method_list:
+            print("availabel methods are {}".format(method_list))
+        self.mean = np.mean(X, axis=0)
+        getattr(self, method)(X - self.mean, iter_max)
+
+    def eigen(self, X, *arg):
+        sample_size, n_features = X.shape
+        if sample_size >= n_features:
+            cov = np.cov(X, rowvar=False)
+            values, vectors = np.linalg.eigh(cov)
+            index = n_features - self.n_components
+        else:
+            cov = np.cov(X)
+            values, vectors = np.linalg.eigh(cov)
+            vectors = (X.T @ vectors) / np.sqrt(sample_size * values)
+            index = sample_size - self.n_components
+        self.I = np.eye(self.n_components)
+        if index == 0:
+            self.var = 0
+        else:
+            self.var = np.mean(values[:index])
+
+        self.W = vectors[:, index:].dot(np.sqrt(np.diag(values[index:]) - self.var * self.I))
+        self.__M = self.W.T @ self.W + self.var * self.I
+        self.C = self.W @ self.W.T + self.var * np.eye(n_features)
+        if index == 0:
+            self.Cinv = np.linalg.inv(self.C)
+        else:
+            self.Cinv = np.eye(n_features) / np.sqrt(self.var) - self.W @ np.linalg.inv(self.__M) @ self.W.T / self.var
+
+    def em(self, X, iter_max):
+        self.I = np.eye(self.n_components)
+        self.W = np.eye(np.size(X, 1), self.n_components)
+        self.var = 1.
+        for i in range(iter_max):
+            W = np.copy(self.W)
+            stats = self._expectation(X)
+            self._maximization(X, *stats)
+            if np.allclose(W, self.W):
+                break
+        self.C = self.W @ self.W.T + self.var * np.eye(np.size(X, 1))
+        self.Cinv = np.linalg.inv(self.C)
+
+    def _expectation(self, X):
+        self.__M = self.W.T @ self.W + self.var * self.I
+        Minv = np.linalg.inv(self.__M)
+        Ez = X @ self.W @ Minv
+        Ezz = self.var * Minv + Ez[:, :, None] * Ez[:, None, :]
+        return Ez, Ezz
+
+    def _maximization(self, X, Ez, Ezz):
+        self.W = X.T @ Ez @ np.linalg.inv(np.sum(Ezz, axis=0))
+        self.var = np.mean(
+            np.mean(X ** 2, axis=1)
+            - 2 * np.mean(Ez @ self.W.T * X, axis=1)
+            + np.trace((Ezz @ self.W.T @ self.W).T) / np.size(X, 1))
+
+    def transform(self, X):
+        """
+        project input data into latent space
+        p(Z|X) = N(Z|(X-mu)WMinv, sigma^-2M)
+
+        Parameters
+        ----------
+        X : (sample_size, n_features) ndarray
+            input data
+
+        Returns
+        -------
+        Z : (sample_size, n_components) ndarray
+            projected input data
+        """
+        return np.linalg.solve(self.__M, ((X - self.mean) @ self.W).T).T
+
+    def fit_transform(self, X, method="eigen"):
+        """
+        perform pca and whiten the input data
+
+        Parameters
+        ----------
+        X : (sample_size, n_features) ndarray
+            input data
+
+        Returns
+        -------
+        Z : (sample_size, n_components) ndarray
+            projected input data
+        """
+        self.fit(X, method)
+        return self.transform(X)
+
+    def proba(self, X):
+        """
+        the marginal distribution of the observed variable
+
+        Parameters
+        ----------
+        X : (sample_size, n_features) ndarray
+            input data
+
+        Returns
+        -------
+        p : (sample_size,) ndarray
+            value of the marginal distribution
+        """
+        d = X - self.mean
+        return (
+            np.exp(-0.5 * np.sum(d @ self.Cinv * d, axis=-1))
+            / np.sqrt(np.linalg.det(self.C))
+            / np.power(2 * np.pi, 0.5 * np.size(X, 1)))

books/PRML/PRML-master-Python/prml/kernel/__init__.py

@@ -0,0 +1,19 @@
+from prml.kernel.polynomial import PolynomialKernel
+from prml.kernel.rbf import RBF
+
+from prml.kernel.gaussian_process_classifier import GaussianProcessClassifier
+from prml.kernel.gaussian_process_regressor import GaussianProcessRegressor
+from prml.kernel.relevance_vector_classifier import RelevanceVectorClassifier
+from prml.kernel.relevance_vector_regressor import RelevanceVectorRegressor
+from prml.kernel.support_vector_classifier import SupportVectorClassifier
+
+
+__all__ = [
+    "PolynomialKernel",
+    "RBF",
+    "GaussianProcessClassifier",
+    "GaussianProcessRegressor",
+    "RelevanceVectorClassifier",
+    "RelevanceVectorRegressor",
+    "SupportVectorClassifier"
+]

books/PRML/PRML-master-Python/prml/kernel/gaussian_process_classifier.py

@@ -0,0 +1,37 @@
+import numpy as np
+
+
+class GaussianProcessClassifier(object):
+
+    def __init__(self, kernel, noise_level=1e-4):
+        """
+        construct gaussian process classifier
+
+        Parameters
+        ----------
+        kernel
+            kernel function to be used to compute Gram matrix
+        noise_level : float
+            parameter to ensure the matrix to be positive
+        """
+        self.kernel = kernel
+        self.noise_level = noise_level
+
+    def _sigmoid(self, a):
+        return np.tanh(a * 0.5) * 0.5 + 0.5
+
+    def fit(self, X, t):
+        if X.ndim == 1:
+            X = X[:, None]
+        self.X = X
+        self.t = t
+        Gram = self.kernel(X, X)
+        self.covariance = Gram + np.eye(len(Gram)) * self.noise_level
+        self.precision = np.linalg.inv(self.covariance)
+
+    def predict(self, X):
+        if X.ndim == 1:
+            X = X[:, None]
+        K = self.kernel(X, self.X)
+        a_mean = K @ self.precision @ self.t
+        return self._sigmoid(a_mean)

books/PRML/PRML-master-Python/prml/kernel/gaussian_process_regressor.py

@@ -0,0 +1,105 @@
+import numpy as np
+
+
+class GaussianProcessRegressor(object):
+
+    def __init__(self, kernel, beta=1.):
+        """
+        construct gaussian process regressor
+
+        Parameters
+        ----------
+        kernel
+            kernel function
+        beta : float
+            precision parameter of observation noise
+        """
+        self.kernel = kernel
+        self.beta = beta
+
+    def fit(self, X, t, iter_max=0, learning_rate=0.1):
+        """
+        maximum likelihood estimation of parameters in kernel function
+
+        Parameters
+        ----------
+        X : ndarray (sample_size, n_features)
+            input
+        t : ndarray (sample_size,)
+            corresponding target
+        iter_max : int
+            maximum number of iterations updating hyperparameters
+        learning_rate : float
+            updation coefficient
+
+        Attributes
+        ----------
+        covariance : ndarray (sample_size, sample_size)
+            variance covariance matrix of gaussian process
+        precision : ndarray (sample_size, sample_size)
+            precision matrix of gaussian process
+
+        Returns
+        -------
+        log_likelihood_list : list
+            list of log likelihood value at each iteration
+        """
+        if X.ndim == 1:
+            X = X[:, None]
+        log_likelihood_list = [-np.Inf]
+        self.X = X
+        self.t = t
+        I = np.eye(len(X))
+        Gram = self.kernel(X, X)
+        self.covariance = Gram + I / self.beta
+        self.precision = np.linalg.inv(self.covariance)
+        for i in range(iter_max):
+            gradients = self.kernel.derivatives(X, X)
+            updates = np.array(
+                [-np.trace(self.precision.dot(grad)) + t.dot(self.precision.dot(grad).dot(self.precision).dot(t)) for grad in gradients])
+            for j in range(iter_max):
+                self.kernel.update_parameters(learning_rate * updates)
+                Gram = self.kernel(X, X)
+                self.covariance = Gram + I / self.beta
+                self.precision = np.linalg.inv(self.covariance)
+                log_like = self.log_likelihood()
+                if log_like > log_likelihood_list[-1]:
+                    log_likelihood_list.append(log_like)
+                    break
+                else:
+                    self.kernel.update_parameters(-learning_rate * updates)
+                    learning_rate *= 0.9
+        log_likelihood_list.pop(0)
+        return log_likelihood_list
+
+    def log_likelihood(self):
+        return -0.5 * (
+            np.linalg.slogdet(self.covariance)[1]
+            + self.t @ self.precision @ self.t
+            + len(self.t) * np.log(2 * np.pi))
+
+    def predict(self, X, with_error=False):
+        """
+        mean of the gaussian process
+
+        Parameters
+        ----------
+        X : ndarray (sample_size, n_features)
+            input
+
+        Returns
+        -------
+        mean : ndarray (sample_size,)
+            predictions of corresponding inputs
+        """
+        if X.ndim == 1:
+            X = X[:, None]
+        K = self.kernel(X, self.X)
+        mean = K @ self.precision @ self.t
+        if with_error:
+            var = (
+                self.kernel(X, X, False)
+                + 1 / self.beta
+                - np.sum(K @ self.precision * K, axis=1))
+            return mean.ravel(), np.sqrt(var.ravel())
+        return mean

books/PRML/PRML-master-Python/prml/kernel/kernel.py

@@ -0,0 +1,28 @@
+import numpy as np
+
+
+class Kernel(object):
+    """
+    Base class for kernel function
+    """
+
+    def _pairwise(self, x, y):
+        """
+        all pairs of x and y
+
+        Parameters
+        ----------
+        x : (sample_size, n_features)
+            input
+        y : (sample_size, n_features)
+            another input
+
+        Returns
+        -------
+        output : tuple
+            two array with shape (sample_size, sample_size, n_features)
+        """
+        return (
+            np.tile(x, (len(y), 1, 1)).transpose(1, 0, 2),
+            np.tile(y, (len(x), 1, 1))
+        )

books/PRML/PRML-master-Python/prml/kernel/polynomial.py

@@ -0,0 +1,43 @@
+import numpy as np
+from prml.kernel.kernel import Kernel
+
+
+class PolynomialKernel(Kernel):
+    """
+    Polynomial kernel
+    k(x,y) = (x @ y + c)^M
+    """
+
+    def __init__(self, degree=2, const=0.):
+        """
+        construct Polynomial kernel
+
+        Parameters
+        ----------
+        const : float
+            a constant to be added
+        degree : int
+            degree of polynomial order
+        """
+        self.const = const
+        self.degree = degree
+
+    def __call__(self, x, y, pairwise=True):
+        """
+        calculate pairwise polynomial kernel
+
+        Parameters
+        ----------
+        x : (..., ndim) ndarray
+            input
+        y : (..., ndim) ndarray
+            another input with the same shape
+
+        Returns
+        -------
+        output : ndarray
+            polynomial kernel
+        """
+        if pairwise:
+            x, y = self._pairwise(x, y)
+        return (np.sum(x * y, axis=-1) + self.const) ** self.degree

books/PRML/PRML-master-Python/prml/kernel/rbf.py

@@ -0,0 +1,58 @@
+import numpy as np
+from prml.kernel.kernel import Kernel
+
+
+class RBF(Kernel):
+
+    def __init__(self, params):
+        """
+        construct Radial basis kernel function
+
+        Parameters
+        ----------
+        params : (ndim + 1,) ndarray
+            parameters of radial basis function
+
+        Attributes
+        ----------
+        ndim : int
+            dimension of expected input data
+        """
+        assert params.ndim == 1
+        self.params = params
+        self.ndim = len(params) - 1
+
+    def __call__(self, x, y, pairwise=True):
+        """
+        calculate radial basis function
+        k(x, y) = c0 * exp(-0.5 * c1 * (x1 - y1) ** 2 ...)
+
+        Parameters
+        ----------
+        x : ndarray [..., ndim]
+            input of this kernel function
+        y : ndarray [..., ndim]
+            another input
+
+        Returns
+        -------
+        output : ndarray
+            output of this radial basis function
+        """
+        assert x.shape[-1] == self.ndim
+        assert y.shape[-1] == self.ndim
+        if pairwise:
+            x, y = self._pairwise(x, y)
+        d = self.params[1:] * (x - y) ** 2
+        return self.params[0] * np.exp(-0.5 * np.sum(d, axis=-1))
+
+    def derivatives(self, x, y, pairwise=True):
+        if pairwise:
+            x, y = self._pairwise(x, y)
+        d = self.params[1:] * (x - y) ** 2
+        delta = np.exp(-0.5 * np.sum(d, axis=-1))
+        deltas = -0.5 * (x - y) ** 2 * (delta * self.params[0])[:, :, None]
+        return np.concatenate((np.expand_dims(delta, 0), deltas.T))
+
+    def update_parameters(self, updates):
+        self.params += updates

books/PRML/PRML-master-Python/prml/kernel/relevance_vector_classifier.py

@@ -0,0 +1,122 @@
+import numpy as np
+
+
+class RelevanceVectorClassifier(object):
+
+    def __init__(self, kernel, alpha=1.):
+        """
+        construct relevance vector classifier
+
+        Parameters
+        ----------
+        kernel : Kernel
+            kernel function to compute components of feature vectors
+        alpha : float
+            initial precision of prior weight distribution
+        """
+        self.kernel = kernel
+        self.alpha = alpha
+
+    def _sigmoid(self, a):
+        return np.tanh(a * 0.5) * 0.5 + 0.5
+
+    def _map_estimate(self, X, t, w, n_iter=10):
+        for _ in range(n_iter):
+            y = self._sigmoid(X @ w)
+            g = X.T @ (y - t) + self.alpha * w
+            H = (X.T * y * (1 - y)) @ X + np.diag(self.alpha)
+            w -= np.linalg.solve(H, g)
+        return w, np.linalg.inv(H)
+
+    def fit(self, X, t, iter_max=100):
+        """
+        maximize evidence with respect ot hyperparameter
+
+        Parameters
+        ----------
+        X : (sample_size, n_features) ndarray
+            input
+        t : (sample_size,) ndarray
+            corresponding target
+        iter_max : int
+            maximum number of iterations
+
+        Attributes
+        ----------
+        X : (N, n_features) ndarray
+            relevance vector
+        t : (N,) ndarray
+            corresponding target
+        alpha : (N,) ndarray
+            hyperparameter for each weight or training sample
+        cov : (N, N) ndarray
+            covariance matrix of weight
+        mean : (N,) ndarray
+            mean of each weight
+        """
+        if X.ndim == 1:
+            X = X[:, None]
+        assert X.ndim == 2
+        assert t.ndim == 1
+        Phi = self.kernel(X, X)
+        N = len(t)
+        self.alpha = np.zeros(N) + self.alpha
+        mean = np.zeros(N)
+        for _ in range(iter_max):
+            param = np.copy(self.alpha)
+            mean, cov = self._map_estimate(Phi, t, mean, 10)
+            gamma = 1 - self.alpha * np.diag(cov)
+            self.alpha = gamma / np.square(mean)
+            np.clip(self.alpha, 0, 1e10, out=self.alpha)
+            if np.allclose(param, self.alpha):
+                break
+        mask = self.alpha < 1e8
+        self.X = X[mask]
+        self.t = t[mask]
+        self.alpha = self.alpha[mask]
+        Phi = self.kernel(self.X, self.X)
+        mean = mean[mask]
+        self.mean, self.covariance = self._map_estimate(Phi, self.t, mean, 100)
+
+    def predict(self, X):
+        """
+        predict class label
+
+        Parameters
+        ----------
+        X : (sample_size, n_features)
+            input
+
+        Returns
+        -------
+        label : (sample_size,) ndarray
+            predicted label
+        """
+        if X.ndim == 1:
+            X = X[:, None]
+        assert X.ndim == 2
+        phi = self.kernel(X, self.X)
+        label = (phi @ self.mean > 0).astype(np.int)
+        return label
+
+    def predict_proba(self, X):
+        """
+        probability of input belonging class one
+
+        Parameters
+        ----------
+        X : (sample_size, n_features) ndarray
+            input
+
+        Returns
+        -------
+        proba : (sample_size,) ndarray
+            probability of predictive distribution p(C1|x)
+        """
+        if X.ndim == 1:
+            X = X[:, None]
+        assert X.ndim == 2
+        phi = self.kernel(X, self.X)
+        mu_a = phi @ self.mean
+        var_a = np.sum(phi @ self.covariance * phi, axis=1)
+        return self._sigmoid(mu_a / np.sqrt(1 + np.pi * var_a / 8))

books/PRML/PRML-master-Python/prml/kernel/relevance_vector_regressor.py

@@ -0,0 +1,102 @@
+import numpy as np
+
+
+class RelevanceVectorRegressor(object):
+
+    def __init__(self, kernel, alpha=1., beta=1.):
+        """
+        construct relevance vector regressor
+
+        Parameters
+        ----------
+        kernel : Kernel
+            kernel function to compute components of feature vectors
+        alpha : float
+            initial precision of prior weight distribution
+        beta : float
+            precision of observation
+        """
+        self.kernel = kernel
+        self.alpha = alpha
+        self.beta = beta
+
+    def fit(self, X, t, iter_max=1000):
+        """
+        maximize evidence with respect to hyperparameter
+
+        Parameters
+        ----------
+        X : (sample_size, n_features) ndarray
+            input
+        t : (sample_size,) ndarray
+            corresponding target
+        iter_max : int
+            maximum number of iterations
+
+        Attributes
+        -------
+        X : (N, n_features) ndarray
+            relevance vector
+        t : (N,) ndarray
+            corresponding target
+        alpha : (N,) ndarray
+            hyperparameter for each weight or training sample
+        cov : (N, N) ndarray
+            covariance matrix of weight
+        mean : (N,) ndarray
+            mean of each weight
+        """
+        if X.ndim == 1:
+            X = X[:, None]
+        assert X.ndim == 2
+        assert t.ndim == 1
+        N = len(t)
+        Phi = self.kernel(X, X)
+        self.alpha = np.zeros(N) + self.alpha
+        for _ in range(iter_max):
+            params = np.hstack([self.alpha, self.beta])
+            precision = np.diag(self.alpha) + self.beta * Phi.T @ Phi
+            covariance = np.linalg.inv(precision)
+            mean = self.beta * covariance @ Phi.T @ t
+            gamma = 1 - self.alpha * np.diag(covariance)
+            self.alpha = gamma / np.square(mean)
+            np.clip(self.alpha, 0, 1e10, out=self.alpha)
+            self.beta = (N - np.sum(gamma)) / np.sum((t - Phi.dot(mean)) ** 2)
+            if np.allclose(params, np.hstack([self.alpha, self.beta])):
+                break
+        mask = self.alpha < 1e9
+        self.X = X[mask]
+        self.t = t[mask]
+        self.alpha = self.alpha[mask]
+        Phi = self.kernel(self.X, self.X)
+        precision = np.diag(self.alpha) + self.beta * Phi.T @ Phi
+        self.covariance = np.linalg.inv(precision)
+        self.mean = self.beta * self.covariance @ Phi.T @ self.t
+
+    def predict(self, X, with_error=True):
+        """
+        predict output with this model
+
+        Parameters
+        ----------
+        X : (sample_size, n_features)
+            input
+        with_error : bool
+            if True, predict with standard deviation of the outputs
+
+        Returns
+        -------
+        mean : (sample_size,) ndarray
+            mean of predictive distribution
+        std : (sample_size,) ndarray
+            standard deviation of predictive distribution
+        """
+        if X.ndim == 1:
+            X = X[:, None]
+        assert X.ndim == 2
+        phi = self.kernel(X, self.X)
+        mean = phi @ self.mean
+        if with_error:
+            var = 1 / self.beta + np.sum(phi @ self.covariance * phi, axis=1)
+            return mean, np.sqrt(var)
+        return mean

books/PRML/PRML-master-Python/prml/kernel/support_vector_classifier.py

@@ -0,0 +1,107 @@
+import numpy as np
+
+
+class SupportVectorClassifier(object):
+
+    def __init__(self, kernel, C=np.Inf):
+        """
+        construct support vector classifier
+
+        Parameters
+        ----------
+        kernel : Kernel
+            kernel function to compute inner products
+        C : float
+            penalty of misclassification
+        """
+        self.kernel = kernel
+        self.C = C
+
+    def fit(self, X:np.ndarray, t:np.ndarray, tol:float=1e-8):
+        """
+        estimate support vectors and their parameters
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            training independent variable
+        t : (N,) np.ndarray
+            training dependent variable
+            binary -1 or 1
+        tol : float, optional
+            numerical tolerance (the default is 1e-8)
+        """
+
+        N = len(t)
+        coef = np.zeros(N)
+        grad = np.ones(N)
+        Gram = self.kernel(X, X)
+
+        while True:
+            tg = t * grad
+            mask_up = (t == 1) & (coef < self.C - tol)
+            mask_up |= (t == -1) & (coef > tol)
+            mask_down = (t == -1) & (coef < self.C - tol)
+            mask_down |= (t == 1) & (coef > tol)
+            i = np.where(mask_up)[0][np.argmax(tg[mask_up])]
+            j = np.where(mask_down)[0][np.argmin(tg[mask_down])]
+            if tg[i] < tg[j] + tol:
+                self.b = 0.5 * (tg[i] + tg[j])
+                break
+            else:
+                A = self.C - coef[i] if t[i] == 1 else coef[i]
+                B = coef[j] if t[j] == 1 else self.C - coef[j]
+                direction = (tg[i] - tg[j]) / (Gram[i, i] - 2 * Gram[i, j] + Gram[j, j])
+                direction = min(A, B, direction)
+                coef[i] += direction * t[i]
+                coef[j] -= direction * t[j]
+                grad -= direction * t * (Gram[i] - Gram[j])
+        support_mask = coef > tol
+        self.a = coef[support_mask]
+        self.X = X[support_mask]
+        self.t = t[support_mask]
+
+    def lagrangian_function(self):
+        return (
+            np.sum(self.a)
+            - self.a
+            @ (self.t * self.t[:, None] * self.kernel(self.X, self.X))
+            @ self.a)
+
+    def predict(self, x):
+        """
+        predict labels of the input
+
+        Parameters
+        ----------
+        x : (sample_size, n_features) ndarray
+            input
+
+        Returns
+        -------
+        label : (sample_size,) ndarray
+            predicted labels
+        """
+        y = self.distance(x)
+        label = np.sign(y)
+        return label
+
+    def distance(self, x):
+        """
+        calculate distance from the decision boundary
+
+        Parameters
+        ----------
+        x : (sample_size, n_features) ndarray
+            input
+
+        Returns
+        -------
+        distance : (sample_size,) ndarray
+            distance from the boundary
+        """
+        distance = np.sum(
+            self.a * self.t
+            * self.kernel(x, self.X),
+            axis=-1) + self.b
+        return distance

books/PRML/PRML-master-Python/prml/linear/__init__.py

@@ -0,0 +1,28 @@
+from prml.linear.bayesian_logistic_regression import BayesianLogisticRegression
+from prml.linear.bayesian_regression import BayesianRegression
+from prml.linear.emprical_bayes_regression import EmpiricalBayesRegression
+from prml.linear.least_squares_classifier import LeastSquaresClassifier
+from prml.linear.linear_regression import LinearRegression
+from prml.linear.fishers_linear_discriminant import FishersLinearDiscriminant
+from prml.linear.logistic_regression import LogisticRegression
+from prml.linear.perceptron import Perceptron
+from prml.linear.ridge_regression import RidgeRegression
+from prml.linear.softmax_regression import SoftmaxRegression
+from prml.linear.variational_linear_regression import VariationalLinearRegression
+from prml.linear.variational_logistic_regression import VariationalLogisticRegression
+
+
+__all__ = [
+    "BayesianLogisticRegression",
+    "BayesianRegression",
+    "EmpiricalBayesRegression",
+    "LeastSquaresClassifier",
+    "LinearRegression",
+    "FishersLinearDiscriminant",
+    "LogisticRegression",
+    "Perceptron",
+    "RidgeRegression",
+    "SoftmaxRegression",
+    "VariationalLinearRegression",
+    "VariationalLogisticRegression"
+]

books/PRML/PRML-master-Python/prml/linear/bayesian_logistic_regression.py

@@ -0,0 +1,66 @@
+import numpy as np
+from prml.linear.logistic_regression import LogisticRegression
+
+
+class BayesianLogisticRegression(LogisticRegression):
+    """
+    Logistic regression model
+
+    w ~ Gaussian(0, alpha^(-1)I)
+    y = sigmoid(X @ w)
+    t ~ Bernoulli(t|y)
+    """
+
+    def __init__(self, alpha:float=1.):
+        self.alpha = alpha
+
+    def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100):
+        """
+        bayesian estimation of logistic regression model
+        using Laplace approximation
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            training data independent variable
+        t : (N,) np.ndarray
+            training data dependent variable
+            binary 0 or 1
+        max_iter : int, optional
+            maximum number of paramter update iteration (the default is 100)
+        """
+        w = np.zeros(np.size(X, 1))
+        eye = np.eye(np.size(X, 1))
+        self.w_mean = np.copy(w)
+        self.w_precision = self.alpha * eye
+        for _ in range(max_iter):
+            w_prev = np.copy(w)
+            y = self._sigmoid(X @ w)
+            grad = X.T @ (y - t) + self.w_precision @ (w - self.w_mean)
+            hessian = (X.T * y * (1 - y)) @ X + self.w_precision
+            try:
+                w -= np.linalg.solve(hessian, grad)
+            except np.linalg.LinAlgError:
+                break
+            if np.allclose(w, w_prev):
+                break
+        self.w_mean = w
+        self.w_precision = hessian
+
+    def proba(self, X:np.ndarray):
+        """
+        compute probability of input belonging class 1
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            training data independent variable
+
+        Returns
+        -------
+        (N,) np.ndarray
+            probability of positive
+        """
+        mu_a = X @ self.w_mean
+        var_a = np.sum(np.linalg.solve(self.w_precision, X.T).T * X, axis=1)
+        return self._sigmoid(mu_a / np.sqrt(1 + np.pi * var_a / 8))

books/PRML/PRML-master-Python/prml/linear/bayesian_regression.py

@@ -0,0 +1,87 @@
+import numpy as np
+from prml.linear.regression import Regression
+
+
+class BayesianRegression(Regression):
+    """
+    Bayesian regression model
+
+    w ~ N(w|0, alpha^(-1)I)
+    y = X @ w
+    t ~ N(t|X @ w, beta^(-1))
+    """
+
+    def __init__(self, alpha:float=1., beta:float=1.):
+        self.alpha = alpha
+        self.beta = beta
+        self.w_mean = None
+        self.w_precision = None
+
+    def _is_prior_defined(self) -> bool:
+        return self.w_mean is not None and self.w_precision is not None
+
+    def _get_prior(self, ndim:int) -> tuple:
+        if self._is_prior_defined():
+            return self.w_mean, self.w_precision
+        else:
+            return np.zeros(ndim), self.alpha * np.eye(ndim)
+
+    def fit(self, X:np.ndarray, t:np.ndarray):
+        """
+        bayesian update of parameters given training dataset
+
+        Parameters
+        ----------
+        X : (N, n_features) np.ndarray
+            training data independent variable
+        t : (N,) np.ndarray
+            training data dependent variable
+        """
+
+        mean_prev, precision_prev = self._get_prior(np.size(X, 1))
+
+        w_precision = precision_prev + self.beta * X.T @ X
+        w_mean = np.linalg.solve(
+            w_precision,
+            precision_prev @ mean_prev + self.beta * X.T @ t
+        )
+        self.w_mean = w_mean
+        self.w_precision = w_precision
+        self.w_cov = np.linalg.inv(self.w_precision)
+
+    def predict(self, X:np.ndarray, return_std:bool=False, sample_size:int=None):
+        """
+        return mean (and standard deviation) of predictive distribution
+
+        Parameters
+        ----------
+        X : (N, n_features) np.ndarray
+            independent variable
+        return_std : bool, optional
+            flag to return standard deviation (the default is False)
+        sample_size : int, optional
+            number of samples to draw from the predictive distribution
+            (the default is None, no sampling from the distribution)
+
+        Returns
+        -------
+        y : (N,) np.ndarray
+            mean of the predictive distribution
+        y_std : (N,) np.ndarray
+            standard deviation of the predictive distribution
+        y_sample : (N, sample_size) np.ndarray
+            samples from the predictive distribution
+        """
+
+        if sample_size is not None:
+            w_sample = np.random.multivariate_normal(
+                self.w_mean, self.w_cov, size=sample_size
+            )
+            y_sample = X @ w_sample.T
+            return y_sample
+        y = X @ self.w_mean
+        if return_std:
+            y_var = 1 / self.beta + np.sum(X @ self.w_cov * X, axis=1)
+            y_std = np.sqrt(y_var)
+            return y, y_std
+        return y

books/PRML/PRML-master-Python/prml/linear/classifier.py

@@ -0,0 +1,5 @@
+class Classifier(object):
+    """
+    Base class for classifiers
+    """
+    pass

books/PRML/PRML-master-Python/prml/linear/emprical_bayes_regression.py

@@ -0,0 +1,86 @@
+import numpy as np
+from prml.linear.bayesian_regression import BayesianRegression
+
+
+class EmpiricalBayesRegression(BayesianRegression):
+    """
+    Empirical Bayes Regression model
+    a.k.a.
+    type 2 maximum likelihood,
+    generalized maximum likelihood,
+    evidence approximation
+
+    w ~ N(w|0, alpha^(-1)I)
+    y = X @ w
+    t ~ N(t|X @ w, beta^(-1))
+    evidence function p(t|X,alpha,beta) = S p(t|w;X,beta)p(w|0;alpha) dw
+    """
+
+    def __init__(self, alpha:float=1., beta:float=1.):
+        super().__init__(alpha, beta)
+
+    def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100):
+        """
+        maximization of evidence function with respect to
+        the hyperparameters alpha and beta given training dataset
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            training independent variable
+        t : (N,) np.ndarray
+            training dependent variable
+        max_iter : int
+            maximum number of iteration
+        """
+        M = X.T @ X
+        eigenvalues = np.linalg.eigvalsh(M)
+        eye = np.eye(np.size(X, 1))
+        N = len(t)
+        for _ in range(max_iter):
+            params = [self.alpha, self.beta]
+
+            w_precision = self.alpha * eye + self.beta * X.T @ X
+            w_mean = self.beta * np.linalg.solve(w_precision, X.T @ t)
+
+            gamma = np.sum(eigenvalues / (self.alpha + eigenvalues))
+            self.alpha = float(gamma / np.sum(w_mean ** 2).clip(min=1e-10))
+            self.beta = float(
+                (N - gamma) / np.sum(np.square(t - X @ w_mean))
+            )
+            if np.allclose(params, [self.alpha, self.beta]):
+                break
+        self.w_mean = w_mean
+        self.w_precision = w_precision
+        self.w_cov = np.linalg.inv(w_precision)
+
+    def _log_prior(self, w):
+        return -0.5 * self.alpha * np.sum(w ** 2)
+
+    def _log_likelihood(self, X, t, w):
+        return -0.5 * self.beta * np.square(t - X @ w).sum()
+
+    def _log_posterior(self, X, t, w):
+        return self._log_likelihood(X, t, w) + self._log_prior(w)
+
+    def log_evidence(self, X:np.ndarray, t:np.ndarray):
+        """
+        logarithm or the evidence function
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            indenpendent variable
+        t : (N,) np.ndarray
+            dependent variable
+        Returns
+        -------
+        float
+            log evidence
+        """
+        N = len(t)
+        D = np.size(X, 1)
+        return 0.5 * (
+            D * np.log(self.alpha) + N * np.log(self.beta)
+            - np.linalg.slogdet(self.w_precision)[1] - D * np.log(2 * np.pi)
+        ) + self._log_posterior(X, t, self.w_mean)

books/PRML/PRML-master-Python/prml/linear/fishers_linear_discriminant.py

@@ -0,0 +1,80 @@
+import numpy as np
+from prml.linear.classifier import Classifier
+from prml.rv.gaussian import Gaussian
+
+
+class FishersLinearDiscriminant(Classifier):
+    """
+    Fisher's Linear discriminant model
+    """
+
+    def __init__(self, w:np.ndarray=None, threshold:float=None):
+        self.w = w
+        self.threshold = threshold
+
+    def fit(self, X:np.ndarray, t:np.ndarray):
+        """
+        estimate parameter given training dataset
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            training dataset independent variable
+        t : (N,) np.ndarray
+            training dataset dependent variable
+            binary 0 or 1
+        """
+        X0 = X[t == 0]
+        X1 = X[t == 1]
+        m0 = np.mean(X0, axis=0)
+        m1 = np.mean(X1, axis=0)
+        cov_inclass = np.cov(X0, rowvar=False) + np.cov(X1, rowvar=False)
+        self.w = np.linalg.solve(cov_inclass, m1 - m0)
+        self.w /= np.linalg.norm(self.w).clip(min=1e-10)
+
+        g0 = Gaussian()
+        g0.fit((X0 @ self.w))
+        g1 = Gaussian()
+        g1.fit((X1 @ self.w))
+        root = np.roots([
+            g1.var - g0.var,
+            2 * (g0.var * g1.mu - g1.var * g0.mu),
+            g1.var * g0.mu ** 2 - g0.var * g1.mu ** 2
+            - g1.var * g0.var * np.log(g1.var / g0.var)
+        ])
+        if g0.mu < root[0] < g1.mu or g1.mu < root[0] < g0.mu:
+            self.threshold = root[0]
+        else:
+            self.threshold = root[1]
+
+    def transform(self, X:np.ndarray):
+        """
+        project data
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            independent variable
+
+        Returns
+        -------
+        y : (N,) np.ndarray
+            projected data
+        """
+        return X @ self.w
+
+    def classify(self, X:np.ndarray):
+        """
+        classify input data
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            independent variable to be classified
+
+        Returns
+        -------
+        (N,) np.ndarray
+            binary class for each input
+        """
+        return (X @ self.w > self.threshold).astype(np.int)

books/PRML/PRML-master-Python/prml/linear/least_squares_classifier.py

@@ -0,0 +1,48 @@
+import numpy as np
+from prml.linear.classifier import Classifier
+from prml.preprocess.label_transformer import LabelTransformer
+
+
+class LeastSquaresClassifier(Classifier):
+    """
+    Least squares classifier model
+
+    X : (N, D)
+    W : (D, K)
+    y = argmax_k X @ W
+    """
+
+    def __init__(self, W:np.ndarray=None):
+        self.W = W
+
+    def fit(self, X:np.ndarray, t:np.ndarray):
+        """
+        least squares fitting for classification
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            training independent variable
+        t : (N,) or (N, K) np.ndarray
+            training dependent variable
+            in class index (N,) or one-of-k coding (N,K)
+        """
+        if t.ndim == 1:
+            t = LabelTransformer().encode(t)
+        self.W = np.linalg.pinv(X) @ t
+
+    def classify(self, X:np.ndarray):
+        """
+        classify input data
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            independent variable to be classified
+
+        Returns
+        -------
+        (N,) np.ndarray
+            class index for each input
+        """
+        return np.argmax(X @ self.W, axis=-1)

books/PRML/PRML-master-Python/prml/linear/linear_regression.py

@@ -0,0 +1,48 @@
+import numpy as np
+from prml.linear.regression import Regression
+
+
+class LinearRegression(Regression):
+    """
+    Linear regression model
+    y = X @ w
+    t ~ N(t|X @ w, var)
+    """
+
+    def fit(self, X:np.ndarray, t:np.ndarray):
+        """
+        perform least squares fitting
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            training independent variable
+        t : (N,) np.ndarray
+            training dependent variable
+        """
+        self.w = np.linalg.pinv(X) @ t
+        self.var = np.mean(np.square(X @ self.w - t))
+
+    def predict(self, X:np.ndarray, return_std:bool=False):
+        """
+        make prediction given input
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            samples to predict their output
+        return_std : bool, optional
+            returns standard deviation of each predition if True
+
+        Returns
+        -------
+        y : (N,) np.ndarray
+            prediction of each sample
+        y_std : (N,) np.ndarray
+            standard deviation of each predition
+        """
+        y = X @ self.w
+        if return_std:
+            y_std = np.sqrt(self.var) + np.zeros_like(y)
+            return y, y_std
+        return y

books/PRML/PRML-master-Python/prml/linear/logistic_regression.py

@@ -0,0 +1,77 @@
+import numpy as np
+from prml.linear.classifier import Classifier
+
+
+class LogisticRegression(Classifier):
+    """
+    Logistic regression model
+
+    y = sigmoid(X @ w)
+    t ~ Bernoulli(t|y)
+    """
+
+    @staticmethod
+    def _sigmoid(a):
+        return np.tanh(a * 0.5) * 0.5 + 0.5
+
+    def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100):
+        """
+        maximum likelihood estimation of logistic regression model
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            training data independent variable
+        t : (N,) np.ndarray
+            training data dependent variable
+            binary 0 or 1
+        max_iter : int, optional
+            maximum number of paramter update iteration (the default is 100)
+        """
+        w = np.zeros(np.size(X, 1))
+        for _ in range(max_iter):
+            w_prev = np.copy(w)
+            y = self._sigmoid(X @ w)
+            grad = X.T @ (y - t)
+            hessian = (X.T * y * (1 - y)) @ X
+            try:
+                w -= np.linalg.solve(hessian, grad)
+            except np.linalg.LinAlgError:
+                break
+            if np.allclose(w, w_prev):
+                break
+        self.w = w
+
+    def proba(self, X:np.ndarray):
+        """
+        compute probability of input belonging class 1
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            training data independent variable
+
+        Returns
+        -------
+        (N,) np.ndarray
+            probability of positive
+        """
+        return self._sigmoid(X @ self.w)
+
+    def classify(self, X:np.ndarray, threshold:float=0.5):
+        """
+        classify input data
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            independent variable to be classified
+        threshold : float, optional
+            threshold of binary classification (default is 0.5)
+
+        Returns
+        -------
+        (N,) np.ndarray
+            binary class for each input
+        """
+        return (self.proba(X) > threshold).astype(np.int)

books/PRML/PRML-master-Python/prml/linear/perceptron.py

@@ -0,0 +1,52 @@
+import numpy as np
+from prml.linear.classifier import Classifier
+
+
+class Perceptron(Classifier):
+    """
+    Perceptron model
+    """
+
+    def fit(self, X, t, max_epoch=100):
+        """
+        fit perceptron model on given input pair
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            training independent variable
+        t : (N,)
+            training dependent variable
+            binary -1 or 1
+        max_epoch : int, optional
+            maximum number of epoch (the default is 100)
+        """
+        self.w = np.zeros(np.size(X, 1))
+        for _ in range(max_epoch):
+            N = len(t)
+            index = np.random.permutation(N)
+            X = X[index]
+            t = t[index]
+            for x, label in zip(X, t):
+                self.w += x * label
+                if (X @ self.w * t > 0).all():
+                    break
+            else:
+                continue
+            break
+
+    def classify(self, X):
+        """
+        classify input data
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            independent variable to be classified
+
+        Returns
+        -------
+        (N,) np.ndarray
+            binary class (-1 or 1) for each input
+        """
+        return np.sign(X @ self.w).astype(np.int)

books/PRML/PRML-master-Python/prml/linear/regression.py

@@ -0,0 +1,5 @@
+class Regression(object):
+    """
+    Base class for regressors
+    """
+    pass

books/PRML/PRML-master-Python/prml/linear/ridge_regression.py

@@ -0,0 +1,44 @@
+import numpy as np
+from prml.linear.regression import Regression
+
+
+class RidgeRegression(Regression):
+    """
+    Ridge regression model
+
+    w* = argmin |t - X @ w| + alpha * |w|_2^2
+    """
+
+    def __init__(self, alpha:float=1.):
+        self.alpha = alpha
+
+    def fit(self, X:np.ndarray, t:np.ndarray):
+        """
+        maximum a posteriori estimation of parameter
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            training data independent variable
+        t : (N,) np.ndarray
+            training data dependent variable
+        """
+
+        eye = np.eye(np.size(X, 1))
+        self.w = np.linalg.solve(self.alpha * eye + X.T @ X, X.T @ t)
+
+    def predict(self, X:np.ndarray):
+        """
+        make prediction given input
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            samples to predict their output
+
+        Returns
+        -------
+        (N,) np.ndarray
+            prediction of each input
+        """
+        return X @ self.w

books/PRML/PRML-master-Python/prml/linear/softmax_regression.py

@@ -0,0 +1,83 @@
+import numpy as np
+from prml.linear.classifier import Classifier
+from prml.preprocess.label_transformer import LabelTransformer
+
+
+class SoftmaxRegression(Classifier):
+    """
+    Softmax regression model
+    aka
+    multinomial logistic regression,
+    multiclass logistic regression,
+    maximum entropy classifier.
+
+    y = softmax(X @ W)
+    t ~ Categorical(t|y)
+    """
+
+    @staticmethod
+    def _softmax(a):
+        a_max = np.max(a, axis=-1, keepdims=True)
+        exp_a = np.exp(a - a_max)
+        return exp_a / np.sum(exp_a, axis=-1, keepdims=True)
+
+    def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100, learning_rate:float=0.1):
+        """
+        maximum likelihood estimation of the parameter
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            training independent variable
+        t : (N,) or (N, K) np.ndarray
+            training dependent variable
+            in class index or one-of-k encoding
+        max_iter : int, optional
+            maximum number of iteration (the default is 100)
+        learning_rate : float, optional
+            learning rate of gradient descent (the default is 0.1)
+        """
+        if t.ndim == 1:
+            t = LabelTransformer().encode(t)
+        self.n_classes = np.size(t, 1)
+        W = np.zeros((np.size(X, 1), self.n_classes))
+        for _ in range(max_iter):
+            W_prev = np.copy(W)
+            y = self._softmax(X @ W)
+            grad = X.T @ (y - t)
+            W -= learning_rate * grad
+            if np.allclose(W, W_prev):
+                break
+        self.W = W
+
+    def proba(self, X:np.ndarray):
+        """
+        compute probability of input belonging each class
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            independent variable
+
+        Returns
+        -------
+        (N, K) np.ndarray
+            probability of each class
+        """
+        return self._softmax(X @ self.W)
+
+    def classify(self, X:np.ndarray):
+        """
+        classify input data
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            independent variable to be classified
+
+        Returns
+        -------
+        (N,) np.ndarray
+            class index for each input
+        """
+        return np.argmax(self.proba(X), axis=-1)

books/PRML/PRML-master-Python/prml/linear/variational_linear_regression.py

@@ -0,0 +1,93 @@
+import numpy as np
+from prml.linear.regression import Regression
+
+
+class VariationalLinearRegression(Regression):
+    """
+    variational bayesian estimation of linear regression model
+    p(w,alpha|X,t)
+    ~ q(w)q(alpha)
+    = N(w|w_mean, w_var)Gamma(alpha|a,b)
+
+    Attributes
+    ----------
+    a : float
+        a parameter of variational posterior gamma distribution
+    b : float
+        another parameter of variational posterior gamma distribution
+    w_mean : (n_features,) ndarray
+        mean of variational posterior gaussian distribution
+    w_var : (n_features, n_feautures) ndarray
+        variance of variational posterior gaussian distribution
+    n_iter : int
+        number of iterations performed
+    """
+
+    def __init__(self, beta:float=1., a0:float=1., b0:float=1.):
+        """
+        construct variational linear regressor
+        Parameters
+        ----------
+        beta : float
+            precision of observation noise
+        a0 : float
+            a parameter of prior gamma distribution
+            Gamma(alpha|a0,b0)
+        b0 : float
+            another parameter of prior gamma distribution
+            Gamma(alpha|a0,b0)
+        """
+        self.beta = beta
+        self.a0 = a0
+        self.b0 = b0
+
+    def fit(self, X:np.ndarray, t:np.ndarray, iter_max:int=100):
+        """
+        variational bayesian estimation of parameter
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            training independent variable
+        t : (N,) np.ndarray
+            training dependent variable
+        iter_max : int, optional
+            maximum number of iteration (the default is 100)
+        """
+        D = np.size(X, 1)
+        self.a = self.a0 + 0.5 * D
+        self.b = self.b0
+        I = np.eye(D)
+        for _ in range(iter_max):
+            param = self.b
+            self.w_var = np.linalg.inv(self.a * I / self.b + self.beta * X.T @ X)
+            self.w_mean = self.beta * self.w_var @ X.T @ t
+            self.b = self.b0 + 0.5 * (np.sum(self.w_mean ** 2) + np.trace(self.w_var))
+            if np.allclose(self.b, param):
+                break
+
+    def predict(self, X:np.ndarray, return_std:bool=False):
+        """
+        make prediction of input
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            independent variable
+        return_std : bool, optional
+            return standard deviation of predictive distribution if True
+            (the default is False)
+
+        Returns
+        -------
+        y : (N,) np.ndarray
+            mean of predictive distribution
+        y_std : (N,) np.ndarray
+            standard deviation of predictive distribution
+        """
+        y = X @ self.w_mean
+        if return_std:
+            y_var = 1 / self.beta + np.sum(X @ self.w_var * X, axis=1)
+            y_std = np.sqrt(y_var)
+            return y, y_std
+        return y

books/PRML/PRML-master-Python/prml/linear/variational_logistic_regression.py

@@ -0,0 +1,88 @@
+import numpy as np
+from prml.linear.logistic_regression import LogisticRegression
+
+
+class VariationalLogisticRegression(LogisticRegression):
+
+    def __init__(self, alpha:float=None, a0:float=1., b0:float=1.):
+        """
+        construct variational logistic regressor
+
+        Parameters
+        ----------
+        alpha : float
+            precision parameter of the prior
+            if None, this is also the subject to estimate
+        a0 : float
+            a parameter of hyper prior Gamma dist.
+            Gamma(alpha|a0,b0)
+            if alpha is not None, this argument will be ignored
+        b0 : float
+            another parameter of hyper prior Gamma dist.
+            Gamma(alpha|a0,b0)
+            if alpha is not None, this argument will be ignored
+        """
+        if alpha is not None:
+            self.__alpha = alpha
+        else:
+            self.a0 = a0
+            self.b0 = b0
+
+    def fit(self, X:np.ndarray, t:np.ndarray, iter_max:int=1000):
+        """
+        variational bayesian estimation of the parameter
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            training independent variable
+        t : (N,) np.ndarray
+            training dependent variable
+        iter_max : int, optional
+            maximum number of iteration (the default is 1000)
+        """
+        N, D = X.shape
+        if hasattr(self, "a0"):
+            self.a = self.a0 + 0.5 * D
+        xi = np.random.uniform(-1, 1, size=N)
+        I = np.eye(D)
+        param = np.copy(xi)
+        for _ in range(iter_max):
+            lambda_ = np.tanh(xi) * 0.25 / xi
+            self.w_var = np.linalg.inv(I / self.alpha + 2 * (lambda_ * X.T) @ X)
+            self.w_mean = self.w_var @ np.sum(X.T * (t - 0.5), axis=1)
+            xi = np.sqrt(np.sum(X @ (self.w_var + self.w_mean * self.w_mean[:, None]) * X, axis=-1))
+            if np.allclose(xi, param):
+                break
+            else:
+                param = np.copy(xi)
+
+    @property
+    def alpha(self):
+        if hasattr(self, "__alpha"):
+            return self.__alpha
+        else:
+            try:
+                self.b = self.b0 + 0.5 * (np.sum(self.w_mean ** 2) + np.trace(self.w_var))
+            except AttributeError:
+                self.b = self.b0
+            return self.a / self.b
+
+    def proba(self, X:np.ndarray):
+        """
+        compute probability of input belonging class 1
+
+        Parameters
+        ----------
+        X : (N, D) np.ndarray
+            training data independent variable
+
+        Returns
+        -------
+        (N,) np.ndarray
+            probability of positive
+        """
+        mu_a = X @ self.w_mean
+        var_a = np.sum(X @ self.w_var * X, axis=1)
+        y = self._sigmoid(mu_a / np.sqrt(1 + np.pi * var_a / 8))
+        return y

books/PRML/PRML-master-Python/prml/markov/__init__.py

@@ -0,0 +1,14 @@
+from .categorical_hmm import CategoricalHMM
+from .gaussian_hmm import GaussianHMM
+from prml.markov.kalman import Kalman, kalman_filter, kalman_smoother
+from .particle import Particle
+
+
+__all__ = [
+    "GaussianHMM",
+    "CategoricalHMM",
+    "Kalman",
+    "kalman_filter",
+    "kalman_smoother",
+    "Particle"
+]

books/PRML/PRML-master-Python/prml/markov/categorical_hmm.py

@@ -0,0 +1,65 @@
+import numpy as np
+from .hmm import HiddenMarkovModel
+
+
+class CategoricalHMM(HiddenMarkovModel):
+    """
+    Hidden Markov Model with categorical emission model
+    """
+
+    def __init__(self, initial_proba, transition_proba, means):
+        """
+        construct hidden markov model with categorical emission model
+
+        Parameters
+        ----------
+        initial_proba : (n_hidden,) np.ndarray
+            probability of initial latent state
+        transition_proba : (n_hidden, n_hidden) np.ndarray
+            transition probability matrix
+            (i, j) component denotes the transition probability from i-th to j-th hidden state
+        means : (n_hidden, ndim) np.ndarray
+            mean parameters of categorical distribution
+
+        Returns
+        -------
+        ndim : int
+            number of observation categories
+        n_hidden : int
+            number of hidden states
+        """
+        assert initial_proba.size == transition_proba.shape[0] == transition_proba.shape[1] == means.shape[0]
+        assert np.allclose(means.sum(axis=1), 1)
+        super().__init__(initial_proba, transition_proba)
+        self.ndim = means.shape[1]
+        self.means = means
+
+    def draw(self, n=100):
+        """
+        draw random sequence from this model
+
+        Parameters
+        ----------
+        n : int
+            length of the random sequence
+
+        Returns
+        -------
+        seq : (n,) np.ndarray
+            generated random sequence
+        """
+        hidden_state = np.random.choice(self.n_hidden, p=self.initial_proba)
+        seq = []
+        while len(seq) < n:
+            seq.append(np.random.choice(self.ndim, p=self.means[hidden_state]))
+            hidden_state = np.random.choice(self.n_hidden, p=self.transition_proba[hidden_state])
+        return np.asarray(seq)
+
+    def likelihood(self, X):
+        return self.means[X]
+
+    def maximize(self, seq, p_hidden, p_transition):
+        self.initial_proba = p_hidden[0] / np.sum(p_hidden[0])
+        self.transition_proba = np.sum(p_transition, axis=0) / np.sum(p_transition, axis=(0, 2))
+        x = p_hidden[:, None, :] * (np.eye(self.ndim)[seq])[:, :, None]
+        self.means = np.sum(x, axis=0) / np.sum(p_hidden, axis=0)

books/PRML/PRML-master-Python/prml/markov/gaussian_hmm.py

@@ -0,0 +1,76 @@
+import numpy as np
+from prml.rv import MultivariateGaussian
+from .hmm import HiddenMarkovModel
+
+
+class GaussianHMM(HiddenMarkovModel):
+    """
+    Hidden Markov Model with Gaussian emission model
+    """
+
+    def __init__(self, initial_proba, transition_proba, means, covs):
+        """
+        construct hidden markov model with Gaussian emission model
+
+        Parameters
+        ----------
+        initial_proba : (n_hidden,) np.ndarray or None
+            probability of initial states
+        transition_proba : (n_hidden, n_hidden) np.ndarray or None
+            transition probability matrix
+            (i, j) component denotes the transition probability from i-th to j-th hidden state
+        means : (n_hidden, ndim) np.ndarray
+            mean of each gaussian component
+        covs : (n_hidden, ndim, ndim) np.ndarray
+            covariance matrix of each gaussian component
+
+        Attributes
+        ----------
+        ndim : int
+            dimensionality of observation space
+        n_hidden : int
+            number of hidden states
+        """
+        assert initial_proba.size == transition_proba.shape[0] == transition_proba.shape[1] == means.shape[0] == covs.shape[0]
+        assert means.shape[1] == covs.shape[1] == covs.shape[2]
+        super().__init__(initial_proba, transition_proba)
+        self.ndim = means.shape[1]
+        self.means = means
+        self.covs = covs
+        self.precisions = np.linalg.inv(self.covs)
+        self.gaussians = [MultivariateGaussian(m, cov) for m, cov in zip(means, covs)]
+
+    def draw(self, n=100):
+        """
+        draw random sequence from this model
+
+        Parameters
+        ----------
+        n : int
+            length of the random sequence
+
+        Returns
+        -------
+        seq : (n, ndim) np.ndarray
+            generated random sequence
+        """
+        hidden_state = np.random.choice(self.n_hidden, p=self.initial_proba)
+        seq = []
+        while len(seq) < n:
+            seq.extend(self.gaussians[hidden_state].draw())
+            hidden_state = np.random.choice(self.n_hidden, p=self.transition_proba[hidden_state])
+        return np.asarray(seq)
+
+    def likelihood(self, X):
+        diff = X[:, None, :] - self.means
+        exponents = np.sum(
+            np.einsum('nki,kij->nkj', diff, self.precisions) * diff, axis=-1)
+        return np.exp(-0.5 * exponents) / np.sqrt(np.linalg.det(self.covs) * (2 * np.pi) ** self.ndim)
+
+    def maximize(self, seq, p_hidden, p_transition):
+        self.initial_proba = p_hidden[0] / np.sum(p_hidden[0])
+        self.transition_proba = np.sum(p_transition, axis=0) / np.sum(p_transition, axis=(0, 2))
+        Nk = np.sum(p_hidden, axis=0)
+        self.means = (seq.T @ p_hidden / Nk).T
+        diffs = seq[:, None, :] - self.means
+        self.covs = np.einsum('nki,nkj->kij', diffs, diffs * p_hidden[:, :, None]) / Nk[:, None, None]

books/PRML/PRML-master-Python/prml/markov/hmm.py

@@ -0,0 +1,178 @@
+import numpy as np
+
+
+class HiddenMarkovModel(object):
+    """
+    Base class of Hidden Markov models
+    """
+
+    def __init__(self, initial_proba, transition_proba):
+        """
+        construct hidden markov model
+
+        Parameters
+        ----------
+        initial_proba : (n_hidden,) np.ndarray
+            initial probability of each hidden state
+        transition_proba : (n_hidden, n_hidden) np.ndarray
+            transition probability matrix
+            (i, j) component denotes the transition probability from i-th to j-th hidden state
+
+        Attribute
+        ---------
+        n_hidden : int
+            number of hidden state
+        """
+        self.n_hidden = initial_proba.size
+        self.initial_proba = initial_proba
+        self.transition_proba = transition_proba
+
+    def fit(self, seq, iter_max=100):
+        """
+        perform EM algorithm to estimate parameter of emission model and hidden variables
+
+        Parameters
+        ----------
+        seq : (N, ndim) np.ndarray
+            observed sequence
+        iter_max : int
+            maximum number of EM steps
+
+        Returns
+        -------
+        posterior : (N, n_hidden) np.ndarray
+            posterior distribution of each latent variable
+        """
+        params = np.hstack(
+            (self.initial_proba.ravel(), self.transition_proba.ravel()))
+        for i in range(iter_max):
+            p_hidden, p_transition = self.expect(seq)
+            self.maximize(seq, p_hidden, p_transition)
+            params_new = np.hstack(
+                (self.initial_proba.ravel(), self.transition_proba.ravel()))
+            if np.allclose(params, params_new):
+                break
+            else:
+                params = params_new
+        return self.forward_backward(seq)
+
+    def expect(self, seq):
+        """
+        estimate posterior distributions of hidden states and
+        transition probability between adjacent latent variables
+
+        Parameters
+        ----------
+        seq : (N, ndim) np.ndarray
+            observed sequence
+
+        Returns
+        -------
+        p_hidden : (N, n_hidden) np.ndarray
+            posterior distribution of each hidden variable
+        p_transition : (N - 1, n_hidden, n_hidden) np.ndarray
+            posterior transition probability between adjacent latent variables
+        """
+        likelihood = self.likelihood(seq)
+
+        f = self.initial_proba * likelihood[0]
+        constant = [f.sum()]
+        forward = [f / f.sum()]
+        for like in likelihood[1:]:
+            f = forward[-1] @ self.transition_proba * like
+            constant.append(f.sum())
+            forward.append(f / f.sum())
+        forward = np.asarray(forward)
+        constant = np.asarray(constant)
+
+        backward = [np.ones(self.n_hidden)]
+        for like, c in zip(likelihood[-1:0:-1], constant[-1:0:-1]):
+            backward.insert(0, self.transition_proba @ (like * backward[0]) / c)
+        backward = np.asarray(backward)
+
+        p_hidden = forward * backward
+        p_transition = self.transition_proba * likelihood[1:, None, :] * backward[1:, None, :] * forward[:-1, :, None]
+        return p_hidden, p_transition
+
+    def forward_backward(self, seq):
+        """
+        estimate posterior distributions of hidden states
+
+        Parameters
+        ----------
+        seq : (N, ndim) np.ndarray
+            observed sequence
+
+        Returns
+        -------
+        posterior : (N, n_hidden) np.ndarray
+            posterior distribution of hidden states
+        """
+        likelihood = self.likelihood(seq)
+
+        f = self.initial_proba * likelihood[0]
+        constant = [f.sum()]
+        forward = [f / f.sum()]
+        for like in likelihood[1:]:
+            f = forward[-1] @ self.transition_proba * like
+            constant.append(f.sum())
+            forward.append(f / f.sum())
+
+        backward = [np.ones(self.n_hidden)]
+        for like, c in zip(likelihood[-1:0:-1], constant[-1:0:-1]):
+            backward.insert(0, self.transition_proba @ (like * backward[0]) / c)
+
+        forward = np.asarray(forward)
+        backward = np.asarray(backward)
+        posterior = forward * backward
+        return posterior
+
+    def filtering(self, seq):
+        """
+        bayesian filtering
+
+        Parameters
+        ----------
+        seq : (N, ndim) np.ndarray
+            observed sequence
+
+        Returns
+        -------
+        posterior : (N, n_hidden) np.ndarray
+            posterior distributions of each latent variables
+        """
+        likelihood = self.likelihood(seq)
+        p = self.initial_proba * likelihood[0]
+        posterior = [p / np.sum(p)]
+        for like in likelihood[1:]:
+            p = posterior[-1] @ self.transition_proba * like
+            posterior.append(p / np.sum(p))
+        posterior = np.asarray(posterior)
+        return posterior
+
+    def viterbi(self, seq):
+        """
+        viterbi algorithm (a.k.a. max-sum algorithm)
+
+        Parameters
+        ----------
+        seq : (N, ndim) np.ndarray
+            observed sequence
+
+        Returns
+        -------
+        seq_hid : (N,) np.ndarray
+            the most probable sequence of hidden variables
+        """
+        nll = -np.log(self.likelihood(seq))
+        cost_total = nll[0]
+        from_list = []
+        for i in range(1, len(seq)):
+            cost_temp = cost_total[:, None] - np.log(self.transition_proba) + nll[i]
+            cost_total = np.min(cost_temp, axis=0)
+            index = np.argmin(cost_temp, axis=0)
+            from_list.append(index)
+        seq_hid = [np.argmin(cost_total)]
+        for source in from_list[::-1]:
+            seq_hid.insert(0, source[seq_hid[0]])
+        return seq_hid

books/PRML/PRML-master-Python/prml/markov/kalman.py

@@ -0,0 +1,268 @@
+import numpy as np
+from prml.rv.multivariate_gaussian import MultivariateGaussian as Gaussian
+from prml.markov.state_space_model import StateSpaceModel
+
+
+class Kalman(StateSpaceModel):
+    """
+    A class to perform kalman filtering or smoothing
+    z : internal state
+    x : observation
+
+    z_1 ~ N(z_1|mu_0, P_0)\n
+    z_n ~ N(z_n|A z_n-1, P)\n
+    x_n ~ N(x_n|C z_n, S)
+
+    Parameters
+    ----------
+    system : (Dz, Dz) np.ndarray
+        system matrix aka transition matrix (A)
+    cov_system : (Dz, Dz) np.ndarray
+        covariance matrix of process noise
+    measure : (Dx, Dz) np.ndarray
+        measurement matrix aka observation matrix (C)
+    cov_measure : (Dx, Dx) np.ndarray
+        covariance matrix of measurement noise
+    mu0 : (Dz,) np.ndarray
+        mean parameter of initial hidden variable
+    P0 : (Dz, Dz) np.ndarray
+        covariance parameter of initial hidden variable
+
+    Attributes
+    ----------
+    Dz : int
+        dimensionality of hidden variable
+    Dx : int
+        dimensionality of observed variable
+    """
+
+
+    def __init__(self, system, cov_system, measure, cov_measure, mu0, P0):
+        """
+        construct Kalman model
+
+        z_1 ~ N(z_1|mu_0, P_0)\n
+        z_n ~ N(z_n|A z_n-1, P)\n
+        x_n ~ N(x_n|C z_n, S)
+
+        Parameters
+        ----------
+        system : (Dz, Dz) np.ndarray
+            system matrix aka transition matrix (A)
+        cov_system : (Dz, Dz) np.ndarray
+            covariance matrix of process noise
+        measure : (Dx, Dz) np.ndarray
+            measurement matrix aka observation matrix (C)
+        cov_measure : (Dx, Dx) np.ndarray
+            covariance matrix of measurement noise
+        mu0 : (Dz,) np.ndarray
+            mean parameter of initial hidden variable
+        P0 : (Dz, Dz) np.ndarray
+            covariance parameter of initial hidden variable
+
+        Attributes
+        ----------
+        hidden_mean : list of (Dz,) np.ndarray
+            list of mean of hidden state starting from the given hidden state
+        hidden_cov : list of (Dz, Dz) np.ndarray
+            list of covariance of hidden state starting from the given hidden state
+        """
+        self.system = system
+        self.cov_system = cov_system
+        self.measure = measure
+        self.cov_measure = cov_measure
+
+        self.hidden_mean = [mu0]
+        self.hidden_cov = [P0]
+        self.hidden_cov_predicted = [None]
+
+        self.smoothed_until = -1
+        self.smoothing_gain = [None]
+
+    def predict(self):
+        """
+        predict hidden state at current step given estimate at previous step
+
+        Returns
+        -------
+        tuple ((Dz,) np.ndarray, (Dz, Dz) np.ndarray)
+            tuple of mean and covariance of the estimate at current step
+        """
+        mu_prev, cov_prev = self.hidden_mean[-1], self.hidden_cov[-1]
+        mu = self.system @ mu_prev
+        cov = self.system @ cov_prev @ self.system.T + self.cov_system
+        self.hidden_mean.append(mu)
+        self.hidden_cov.append(cov)
+        self.hidden_cov_predicted.append(np.copy(cov))
+        return mu, cov
+
+    def filter(self, observed):
+        """
+        bayesian update of current estimate given current observation
+
+        Parameters
+        ----------
+        observed : (Dx,) np.ndarray
+            current observation
+
+        Returns
+        -------
+        tuple ((Dz,) np.ndarray, (Dz, Dz) np.ndarray)
+            tuple of mean and covariance of the updated estimate
+        """
+        mu, cov = self.hidden_mean[-1], self.hidden_cov[-1]
+        innovation = observed - self.measure @ mu
+        cov_innovation = self.cov_measure + self.measure @ cov @ self.measure.T
+        kalman_gain = np.linalg.solve(cov_innovation, self.measure @ cov).T
+        mu += kalman_gain @ innovation
+        cov -= kalman_gain @ self.measure @ cov
+        return mu, cov
+
+    def filtering(self, observed_sequence):
+        """
+        perform kalman filtering given observed sequence
+
+        Parameters
+        ----------
+        observed_sequence : (T, Dx) np.ndarray
+            sequence of observations
+
+        Returns
+        -------
+        tuple ((T, Dz) np.ndarray, (T, Dz, Dz) np.ndarray)
+            seuquence of mean and covariance of hidden variable at each time step
+        """
+        for obs in observed_sequence:
+            self.predict()
+            self.filter(obs)
+        mean_sequence = np.asarray(self.hidden_mean[1:])
+        cov_sequence = np.asarray(self.hidden_cov[1:])
+        return mean_sequence, cov_sequence
+
+    def smooth(self):
+        """
+        bayesian update of current estimate with future observations
+        """
+        mean_smoothed_next = self.hidden_mean[self.smoothed_until]
+        cov_smoothed_next = self.hidden_cov[self.smoothed_until]
+        cov_pred_next = self.hidden_cov_predicted[self.smoothed_until]
+
+        self.smoothed_until -= 1
+        mean = self.hidden_mean[self.smoothed_until]
+        cov = self.hidden_cov[self.smoothed_until]
+        gain = np.linalg.solve(cov_pred_next, self.system @ cov).T
+        mean += gain @ (mean_smoothed_next - self.system @ mean)
+        cov += gain @ (cov_smoothed_next - cov_pred_next) @ gain.T
+        self.smoothing_gain.insert(0, gain)
+
+    def smoothing(self, observed_sequence:np.ndarray=None):
+        """
+        perform Kalman smoothing (given observed sequence)
+
+        Parameters
+        ----------
+        observed_sequence : (T, Dx) np.ndarray, optional
+            sequence of observation
+            run Kalman filter if given (the default is None)
+
+        Returns
+        -------
+        tuple ((T, Dz) np.ndarray, (T, Dz, Dz) np.ndarray)
+            sequence of mean and covariance of hidden variable at each time step
+        """
+        if observed_sequence is not None:
+            self.filtering(observed_sequence)
+        while self.smoothed_until != -len(self.hidden_mean):
+            self.smooth()
+        mean_sequence = np.asarray(self.hidden_mean[1:])
+        cov_sequence = np.asarray(self.hidden_cov[1:])
+        return mean_sequence, cov_sequence
+
+    def update_parameter(self, observation_sequence):
+        """
+        maximization step of EM algorithm
+        """
+        mu0 = self.hidden_mean[1]
+        P0 = self.hidden_cov[1]
+
+        Ezn = np.asarray(self.hidden_mean)
+        Eznzn = np.asarray(self.hidden_cov) + Ezn[..., None] * Ezn[:, None, :]
+        Eznzn_1 = np.einsum("nij,nkj->nik", self.hidden_cov[2:], self.smoothing_gain[1:-1]) + Ezn[2:, :, None] * Ezn[1:-1, None, :]
+        self.system = np.linalg.solve(np.sum(Eznzn[2:], axis=0), np.sum(Eznzn_1, axis=0).T).T
+        self.cov_system = np.mean(
+            Eznzn[2:]
+            - np.einsum("ij,nkj->nik", self.system, Eznzn_1)
+            - np.einsum("nij,kj->nik", Eznzn_1, self.system)
+            + np.einsum("ij,njk,lk->nil", self.system, Eznzn[1:-1], self.system),
+            axis=0
+        )
+        self.measure = np.linalg.solve(
+            np.sum(Eznzn[1:], axis=0),
+            np.sum(np.einsum("ni,nj->nij", Ezn[1:], observation_sequence), axis=0)
+        ).T
+        self.cov_measure = np.mean(
+            np.einsum("ni,nj->nij", observation_sequence, observation_sequence)
+            - np.einsum("ij,nj,nk->nik", self.measure, Ezn[1:], observation_sequence)
+            - np.einsum("ni,nj,kj->nik", observation_sequence, Ezn[1:], self.measure)
+            + np.einsum("ij,njk,lk->nil", self.measure, Eznzn[1:], self.measure),
+            axis=0
+        )
+        return self.system, self.cov_system, self.measure, self.cov_measure, mu0, P0
+
+    def fit(self, sequence, max_iter=10):
+        for _ in range(max_iter):
+            kalman_smoother(self, sequence)
+            param = self.update_parameter(sequence)
+            self.__init__(*param)
+        return kalman_smoother(self, sequence)
+
+
+def kalman_filter(kalman:Kalman, observed_sequence:np.ndarray)->tuple:
+    """
+    perform kalman filtering given Kalman model and observed sequence
+
+    Parameters
+    ----------
+    kalman : Kalman
+        Kalman model
+    observed_sequence : (T, Dx) np.ndarray
+        sequence of observations
+
+    Returns
+    -------
+    tuple ((T, Dz) np.ndarray, (T, Dz, Dz) np.ndarray)
+        seuquence of mean and covariance of hidden variable at each time step
+    """
+    for obs in observed_sequence:
+        kalman.predict()
+        kalman.filter(obs)
+    mean_sequence = np.asarray(kalman.hidden_mean[1:])
+    cov_sequence = np.asarray(kalman.hidden_cov[1:])
+    return mean_sequence, cov_sequence
+
+
+def kalman_smoother(kalman:Kalman, observed_sequence:np.ndarray=None):
+    """
+    perform Kalman smoothing given Kalman model (and observed sequence)
+
+    Parameters
+    ----------
+    kalman : Kalman
+        Kalman model
+    observed_sequence : (T, Dx) np.ndarray, optional
+        sequence of observation
+        run Kalman filter if given (the default is None)
+
+    Returns
+    -------
+    tuple ((T, Dz) np.ndarray, (T, Dz, Dz) np.ndarray)
+        seuqnce of mean and covariance of hidden variable at each time step
+    """
+
+    if observed_sequence is not None:
+        kalman_filter(kalman, observed_sequence)
+    while kalman.smoothed_until != -len(kalman.hidden_mean):
+        kalman.smooth()
+    mean_sequence = np.asarray(kalman.hidden_mean[1:])
+    cov_sequence = np.asarray(kalman.hidden_cov[1:])
+    return mean_sequence, cov_sequence

books/PRML/PRML-master-Python/prml/markov/particle.py

@@ -0,0 +1,124 @@
+import numpy as np
+from scipy.misc import logsumexp
+from scipy.spatial.distance import cdist
+from .state_space_model import StateSpaceModel
+
+
+class Particle(StateSpaceModel):
+    """
+    A class to perform particle filtering, smoothing
+
+    z_1 ~ p(z_1)\n
+    z_n ~ p(z_n|z_n-1)\n
+    x_n ~ p(x_n|z_n)
+
+    Parameters
+    ----------
+    init_particle : (n_particle, ndim_hidden)
+        initial hidden state
+    sampler : callable (particles)
+        function to sample particles at current step given previous state
+    nll : callable (observation, particles)
+        function to compute negative log likelihood for each particle
+
+    Attribute
+    ---------
+    hidden_state : list of (n_paticle, ndim_hidden) np.ndarray
+        list of particles
+    """
+
+    def __init__(self, init_particle, system, cov_system, nll, pdf=None):
+        """
+        construct state space model to perform particle filtering or smoothing
+
+        Parameters
+        ----------
+        init_particle : (n_particle, ndim_hidden) np.ndarray
+            initial hidden state
+        system : (ndim_hidden, ndim_hidden) np.ndarray
+            system matrix aka transition matrix
+        cov_system : (ndim_hidden, ndim_hidden) np.ndarray
+            covariance matrix of process noise
+        nll : callable (observation, particles)
+            function to compute negative log likelihood for each particle
+
+        Attribute
+        ---------
+        particle : list of (n_paticle, ndim_hidden) np.ndarray
+            list of particles at each step
+        weight : list of (n_particle,) np.ndarray
+            list of importance of each particle at each step
+        n_particle : int
+            number of particles at each step
+        """
+        self.particle = [init_particle]
+        self.n_particle, self.ndim_hidden = init_particle.shape
+        self.weight = [np.ones(self.n_particle) / self.n_particle]
+        self.system = system
+        self.cov_system = cov_system
+        self.nll = nll
+        self.smoothed_until = -1
+
+    def resample(self):
+        index = np.random.choice(self.n_particle, self.n_particle, p=self.weight[-1])
+        return self.particle[-1][index]
+
+    def predict(self):
+        predicted = self.resample() @ self.system.T
+        predicted += np.random.multivariate_normal(np.zeros(self.ndim_hidden), self.cov_system, self.n_particle)
+        self.particle.append(predicted)
+        self.weight.append(np.ones(self.n_particle) / self.n_particle)
+        return predicted, self.weight[-1]
+
+    def weigh(self, observed):
+        logit = -self.nll(observed, self.particle[-1])
+        logit -= logsumexp(logit)
+        self.weight[-1] = np.exp(logit)
+
+    def filter(self, observed):
+        self.weigh(observed)
+        return self.particle[-1], self.weight[-1]
+
+    def filtering(self, observed_sequence):
+        mean = []
+        cov = []
+        for obs in observed_sequence:
+            self.predict()
+            p, w = self.filter(obs)
+            mean.append(np.average(p, axis=0, weights=w))
+            cov.append(np.cov(p, rowvar=False, aweights=w))
+        return np.asarray(mean), np.asarray(cov)
+
+    def transition_probability(self, particle, particle_prev):
+        dist = cdist(
+            particle,
+            particle_prev @ self.system.T,
+            "mahalanobis",
+            VI=np.linalg.inv(self.cov_system))
+        matrix = np.exp(-0.5 * np.square(dist))
+        matrix /= np.sum(matrix, axis=1, keepdims=True)
+        matrix[np.isnan(matrix)] = 1 / self.n_particle
+        return matrix
+
+    def smooth(self):
+        particle_next = self.particle[self.smoothed_until]
+        weight_next = self.weight[self.smoothed_until]
+
+        self.smoothed_until -= 1
+        particle = self.particle[self.smoothed_until]
+        weight = self.weight[self.smoothed_until]
+        matrix = self.transition_probability(particle_next, particle).T
+        weight *= matrix @ weight_next / (weight @ matrix)
+        weight /= np.sum(weight, keepdims=True)
+
+    def smoothing(self, observed_sequence:np.ndarray=None):
+        if observed_sequence is not None:
+            self.filtering(observed_sequence)
+        while self.smoothed_until != -len(self.particle):
+            self.smooth()
+        mean = []
+        cov = []
+        for p, w in zip(self.particle, self.weight):
+            mean.append(np.average(p, axis=0, weights=w))
+            cov.append(np.cov(p, rowvar=False, aweights=w))
+        return np.asarray(mean), np.asarray(cov)

books/PRML/PRML-master-Python/prml/markov/state_space_model.py

@@ -0,0 +1,5 @@
+class StateSpaceModel(object):
+    """
+    Base class for state-space models
+    """
+    pass

books/PRML/PRML-master-Python/prml/nn/__init__.py

@@ -0,0 +1,34 @@
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.parameter import Parameter
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.array.flatten import flatten
+from prml.nn.array.reshape import reshape
+from prml.nn.array.split import split
+from prml.nn.array.transpose import transpose
+from prml.nn import linalg
+from prml.nn.image.convolve2d import convolve2d
+from prml.nn.image.max_pooling2d import max_pooling2d
+from prml.nn.math.abs import abs
+from prml.nn.math.exp import exp
+from prml.nn.math.gamma import gamma
+from prml.nn.math.log import log
+from prml.nn.math.mean import mean
+from prml.nn.math.power import power
+from prml.nn.math.product import prod
+from prml.nn.math.sqrt import sqrt
+from prml.nn.math.square import square
+from prml.nn.math.sum import sum
+from prml.nn.nonlinear.relu import relu
+from prml.nn.nonlinear.sigmoid import sigmoid
+from prml.nn.nonlinear.softmax import softmax
+from prml.nn.nonlinear.softplus import softplus
+from prml.nn.nonlinear.tanh import tanh
+from prml.nn import optimizer
+from prml.nn import random
+from prml.nn.network import Network
+
+
+__all__ = [
+    "optimizer",
+    "Network"
+]

books/PRML/PRML-master-Python/prml/nn/array/__init__.py

@@ -0,0 +1,19 @@
+from prml.nn.array.broadcast import broadcast_to
+from prml.nn.array.flatten import flatten
+from prml.nn.array.reshape import reshape, reshape_method
+from prml.nn.array.split import split
+from prml.nn.array.transpose import transpose, transpose_method
+from prml.nn.tensor.tensor import Tensor
+
+
+Tensor.flatten = flatten
+Tensor.reshape = reshape_method
+Tensor.transpose = transpose_method
+
+__all__ = [
+    "broadcast_to",
+    "flatten",
+    "reshape",
+    "split",
+    "transpose"
+]

books/PRML/PRML-master-Python/prml/nn/array/broadcast.py

@@ -0,0 +1,36 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class BroadcastTo(Function):
+    """
+    Broadcast a tensor to an new shape
+    """
+
+    def forward(self, x, shape):
+        x = self._convert2tensor(x)
+        self.x = x
+        output = np.broadcast_to(x.value, shape)
+        if isinstance(self.x, Constant):
+            return Constant(output)
+        return Tensor(output, function=self)
+
+    def backward(self, delta):
+        dx = delta
+        if delta.ndim != self.x.ndim:
+            dx = dx.sum(axis=tuple(range(dx.ndim - self.x.ndim)))
+            if isinstance(dx, np.number):
+                dx = np.array(dx)
+        axis = tuple(i for i, len_ in enumerate(self.x.shape) if len_ == 1)
+        if axis:
+            dx = dx.sum(axis=axis, keepdims=True)
+        self.x.backward(dx)
+
+
+def broadcast_to(x, shape):
+    """
+    Broadcast a tensor to an new shape
+    """
+    return BroadcastTo().forward(x, shape)

books/PRML/PRML-master-Python/prml/nn/array/flatten.py

@@ -0,0 +1,28 @@
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Flatten(Function):
+    """
+    flatten array
+    """
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self._atleast_ndim(x, 2)
+        self.x = x
+        if isinstance(self.x, Constant):
+            return Constant(x.value.flatten())
+        return Tensor(x.value.flatten(), function=self)
+
+    def backward(self, delta):
+        dx = delta.reshape(*self.x.shape)
+        self.x.backward(dx)
+
+
+def flatten(x):
+    """
+    flatten N-dimensional array (N >= 2)
+    """
+    return Flatten().forward(x)

books/PRML/PRML-master-Python/prml/nn/array/reshape.py

@@ -0,0 +1,32 @@
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Reshape(Function):
+    """
+    reshape array
+    """
+
+    def forward(self, x, shape):
+        x = self._convert2tensor(x)
+        self._atleast_ndim(x, 1)
+        self.x = x
+        if isinstance(self.x, Constant):
+            return Constant(x.value.reshape(*shape))
+        return Tensor(x.value.reshape(*shape), function=self)
+
+    def backward(self, delta):
+        dx = delta.reshape(*self.x.shape)
+        self.x.backward(dx)
+
+
+def reshape(x, shape):
+    """
+    reshape N-dimensional array (N >= 1)
+    """
+    return Reshape().forward(x, shape)
+
+
+def reshape_method(x, *args):
+    return Reshape().forward(x, args)

books/PRML/PRML-master-Python/prml/nn/array/split.py

@@ -0,0 +1,48 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Nth(Function):
+
+    def __init__(self, n):
+        self.n = n
+
+    def forward(self, x):
+        self.x = x
+        if isinstance(self.x, Constant):
+            return Constant(x.value)
+        return Tensor(x.value, function=self)
+
+    def backward(self, delta):
+        self.x.backward(delta, n=self.n)
+
+
+class Split(Function):
+
+    def __init__(self, indices_or_sections, axis=-1):
+        self.indices_or_sections = indices_or_sections
+        self.axis = axis
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self._atleast_ndim(x, 1)
+        self.x = x
+        output = np.split(x.value, self.indices_or_sections, self.axis)
+        if isinstance(self.x, Constant):
+            return tuple([Constant(out) for out in output])
+        self.n_output = len(output)
+        self.delta = [None for _ in output]
+        return tuple([Tensor(out, function=self) for out in output])
+
+    def backward(self, delta, n):
+        self.delta[n] = delta
+        if all([d is not None for d in self.delta]):
+            dx = np.concatenate(self.delta, axis=self.axis)
+            self.x.backward(dx)
+
+
+def split(x, indices_or_sections, axis=-1):
+    output = Split(indices_or_sections, axis).forward(x)
+    return tuple([Nth(i).forward(out) for i, out in enumerate(output)])

books/PRML/PRML-master-Python/prml/nn/array/transpose.py

@@ -0,0 +1,36 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Transpose(Function):
+
+    def __init__(self, axes=None):
+        self.axes = axes
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        if self.axes is not None:
+            self._equal_ndim(x, len(self.axes))
+        self.x = x
+        if isinstance(self.x, Constant):
+            return Constant(np.transpose(x.value, self.axes))
+        return Tensor(np.transpose(x.value, self.axes), function=self)
+
+    def backward(self, delta):
+        if self.axes is None:
+            dx = np.transpose(delta)
+        else:
+            dx = np.transpose(delta, np.argsort(self.axes))
+        self.x.backward(dx)
+
+
+def transpose(x, axes=None):
+    return Transpose(axes).forward(x)
+
+
+def transpose_method(x, *args):
+    if args == ():
+        args = None
+    return Transpose(args).forward(x)

books/PRML/PRML-master-Python/prml/nn/function.py

@@ -0,0 +1,33 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+
+
+class Function(object):
+    """
+    Base class for differentiable functions
+    """
+
+    def _convert2tensor(self, x):
+        if isinstance(x, (int, float, np.number, np.ndarray)):
+            x = Constant(x)
+        elif not isinstance(x, Tensor):
+            raise TypeError(
+                "Unsupported class for input: {}".format(type(x))
+            )
+        return x
+
+    def _equal_ndim(self, x, ndim):
+        if x.ndim != ndim:
+            raise ValueError(
+                "dimensionality of the input must be {}, not {}"
+                .format(ndim, x.ndim)
+            )
+
+    def _atleast_ndim(self, x, ndim):
+        if x.ndim < ndim:
+            raise ValueError(
+                "dimensionality of the input must be"
+                " larger or equal to {}, not {}"
+                .format(ndim, x.ndim)
+            )

books/PRML/PRML-master-Python/prml/nn/image/__init__.py

@@ -0,0 +1,8 @@
+from prml.nn.image.convolve2d import convolve2d
+from prml.nn.image.max_pooling2d import max_pooling2d
+
+
+__all__ = [
+    "convolve2d",
+    "max_pooling2d"
+]

books/PRML/PRML-master-Python/prml/nn/image/convolve2d.py

@@ -0,0 +1,100 @@
+import numpy as np
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+from prml.nn.image.util import img2patch, patch2img
+
+
+class Convolve2d(Function):
+
+    def __init__(self, stride, pad):
+        """
+        construct 2 dimensional convolution function
+
+        Parameters
+        ----------
+        stride : int or tuple of ints
+            stride of kernel application
+        pad : int or tuple of ints
+            padding image
+        """
+        self.stride = self._check_tuple(stride, "stride")
+        self.pad = self._check_tuple(pad, "pad")
+        self.pad = (0,) + self.pad + (0,)
+
+    def _check_tuple(self, tup, name):
+        if isinstance(tup, int):
+            tup = (tup,) * 2
+        if not isinstance(tup, tuple):
+            raise TypeError(
+                "Unsupported type for {}: {}".format(name, type(tup))
+            )
+        if len(tup) != 2:
+            raise ValueError(
+                "the length of {} must be 2, not {}".format(name, len(tup))
+            )
+        if not all([isinstance(n, int) for n in tup]):
+            raise TypeError(
+                "Unsuported type for {}".format(name)
+            )
+        if not all([n >= 0 for n in tup]):
+            raise ValueError(
+                "{} must be non-negative values".format(name)
+            )
+        return tup
+
+    def _check_input(self, x, y):
+        x = self._convert2tensor(x)
+        y = self._convert2tensor(y)
+        self._equal_ndim(x, 4)
+        self._equal_ndim(y, 4)
+        if x.shape[3] != y.shape[2]:
+            raise ValueError(
+                "shapes {} and {} not aligned: {} (dim 3) != {} (dim 2)"
+                .format(x.shape, y.shape, x.shape[3], y.shape[2])
+            )
+        return x, y
+
+    def forward(self, x, y):
+        x, y = self._check_input(x, y)
+        self.x = x
+        self.y = y
+        img = np.pad(x.value, [(p,) for p in self.pad], "constant")
+        self.shape = img.shape
+        self.patch = img2patch(img, y.shape[:2], self.stride)
+        return Tensor(np.tensordot(self.patch, y.value, axes=((3, 4, 5), (0, 1, 2))), function=self)
+
+    def backward(self, delta):
+        dx = patch2img(
+            np.tensordot(delta, self.y.value, (3, 3)),
+            self.stride,
+            self.shape
+        )
+        slices = [slice(p, len_ - p) for p, len_ in zip(self.pad, self.shape)]
+        dx = dx[slices]
+        dy = np.tensordot(self.patch, delta, axes=((0, 1, 2),) * 2)
+        self.x.backward(dx)
+        self.y.backward(dy)
+
+
+def convolve2d(x, y, stride=1, pad=0):
+    """
+    returns convolution of two tensors
+
+    Parameters
+    ----------
+    x : (n_batch, xlen, ylen, in_channel) Tensor
+        input tensor to be convolved
+    y : (kx, ky, in_channel, out_channel) Tensor
+        convolution kernel
+    stride : int or tuple of ints (sx, sy)
+        stride of kernel application
+    pad : int or tuple of ints (px, py)
+        padding image
+
+    Returns
+    -------
+    output : (n_batch, xlen', ylen', out_channel) Tensor
+        input convolved with kernel
+        len' = (len + p - k) // s + 1
+    """
+    return Convolve2d(stride, pad).forward(x, y)

books/PRML/PRML-master-Python/prml/nn/image/max_pooling2d.py

@@ -0,0 +1,93 @@
+import numpy as np
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+from prml.nn.image.util import img2patch, patch2img
+
+
+class MaxPooling2d(Function):
+
+    def __init__(self, pool_size, stride, pad):
+        """
+        construct 2 dimensional max-pooling function
+
+        Parameters
+        ----------
+        pool_size : int or tuple of ints
+            pooling size
+        stride : int or tuple of ints
+            stride of kernel application
+        pad : int or tuple of ints
+            padding image
+        """
+        self.pool_size = self._check_tuple(pool_size, "pool_size")
+        self.stride = self._check_tuple(stride, "stride")
+        self.pad = self._check_tuple(pad, "pad")
+        self.pad = (0,) + self.pad + (0,)
+
+    def _check_tuple(self, tup, name):
+        if isinstance(tup, int):
+            tup = (tup,) * 2
+        if not isinstance(tup, tuple):
+            raise TypeError(
+                "Unsupported type for {}: {}".format(name, type(tup))
+            )
+        if len(tup) != 2:
+            raise ValueError(
+                "the length of {} must be 2, not {}".format(name, len(tup))
+            )
+        if not all([isinstance(n, int) for n in tup]):
+            raise TypeError(
+                "Unsuported type for {}".format(name)
+            )
+        if not all([n >= 0 for n in tup]):
+            raise ValueError(
+                "{} must be non-negative values".format(name)
+            )
+        return tup
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self._equal_ndim(x, 4)
+        self.x = x
+        img = np.pad(x.value, [(p,) for p in self.pad], "constant")
+        patch = img2patch(img, self.pool_size, self.stride)
+        n_batch, xlen_out, ylen_out, _, _, in_channels = patch.shape
+        patch = patch.reshape(n_batch, xlen_out, ylen_out, -1, in_channels)
+        self.shape = img.shape
+        self.index = patch.argmax(axis=3)
+        return Tensor(patch.max(axis=3), function=self)
+
+    def backward(self, delta):
+        delta_patch = np.zeros(delta.shape + (np.prod(self.pool_size),))
+        index = np.where(delta == delta) + (self.index.ravel(),)
+        delta_patch[index] = delta.ravel()
+        delta_patch = np.reshape(delta_patch, delta.shape + self.pool_size)
+        delta_patch = delta_patch.transpose(0, 1, 2, 4, 5, 3)
+        dx = patch2img(delta_patch, self.stride, self.shape)
+        slices = [slice(p, len_ - p) for p, len_ in zip(self.pad, self.shape)]
+        dx = dx[slices]
+        self.x.backward(dx)
+
+
+def max_pooling2d(x, pool_size, stride=1, pad=0):
+    """
+    spatial max pooling
+
+    Parameters
+    ----------
+    x : (n_batch, xlen, ylen, in_channel) Tensor
+        input tensor
+    pool_size : int or tuple of ints (kx, ky)
+        pooling size
+    stride : int or tuple of ints (sx, sy)
+        stride of pooling application
+    pad : int or tuple of ints (px, py)
+        padding input
+
+    Returns
+    -------
+    output : (n_batch, xlen', ylen', out_channel) Tensor
+        max pooled image
+        len' = (len + p - k) // s + 1
+    """
+    return MaxPooling2d(pool_size, stride, pad).forward(x)

books/PRML/PRML-master-Python/prml/nn/image/util.py

@@ -0,0 +1,62 @@
+import itertools
+import numpy as np
+from numpy.lib.stride_tricks import as_strided
+
+
+def img2patch(img, size, step=1):
+    """
+    convert batch of image array into patches
+    Parameters
+    ----------
+    img : (n_batch, xlen_in, ylen_in, in_channels) ndarray
+        batch of images
+    size : tuple or int
+        patch size
+    step : tuple or int
+        stride of patches
+    Returns
+    -------
+    patch : (n_batch, xlen_out, ylen_out, size, size, in_channels) ndarray
+        batch of patches at all points
+        len_out = (len_in - size) // step + 1
+    """
+    ndim = img.ndim
+    if isinstance(size, int):
+        size = (size,) * (ndim - 2)
+    if isinstance(step, int):
+        step = (step,) * (ndim - 2)
+
+    slices = [slice(None, None, s) for s in step]
+    window_strides = img.strides[1:]
+    index_strides = img[[slice(None)] + slices].strides[:-1]
+
+    out_shape = tuple(
+        np.subtract(img.shape[1: -1], size) // np.array(step) + 1)
+    out_shape = (len(img),) + out_shape + size + (np.size(img, -1),)
+    strides = index_strides + window_strides
+    patch = as_strided(img, shape=out_shape, strides=strides)
+    return patch
+
+
+def patch2img(x, stride, shape):
+    """
+    sum up patches and form an image
+    Parameters
+    ----------
+    x : (n_batch, xlen_in, ylen_in, kx, ky, in_channels) ndarray
+        batch of patches at all points
+    stride : tuple
+        applied stride to take patches
+    shape : (n_batch, xlen_out, ylen_out, in_channels) tuple
+        output image shape
+    Returns
+    -------
+    img : (n_batch, len_out, ylen_out, in_channels) ndarray
+        image
+    """
+    img = np.zeros(shape, dtype=np.float32)
+    kx, ky = x.shape[3: 5]
+    for i, j in itertools.product(range(kx), range(ky)):
+        slices = [slice(b, b + s * len_, s) for b, s, len_ in zip([i, j], stride, x.shape[1: 3])]
+        img[[slice(None)] + slices] += x[..., i, j, :]
+    return img

books/PRML/PRML-master-Python/prml/nn/linalg/__init__.py

@@ -0,0 +1,16 @@
+from prml.nn.linalg.cholesky import cholesky
+from prml.nn.linalg.det import det
+from prml.nn.linalg.inv import inv
+from prml.nn.linalg.logdet import logdet
+from prml.nn.linalg.solve import solve
+from prml.nn.linalg.trace import trace
+
+
+__all__ = [
+    "cholesky",
+    "det",
+    "inv",
+    "logdet",
+    "solve",
+    "trace"
+]

books/PRML/PRML-master-Python/prml/nn/linalg/cholesky.py

@@ -0,0 +1,45 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Cholesky(Function):
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        self.output = np.linalg.cholesky(x.value)
+        if isinstance(self.x, Constant):
+            return Constant(self.output)
+        return Tensor(self.output, function=self)
+
+    def backward(self, delta):
+        delta_lower = np.tril(delta)
+        P = phi(self.output.T @ delta_lower)
+        S = np.linalg.solve(
+            self.output.T,
+            P @ np.linalg.inv(self.output)
+        )
+        dx = S + S.T + np.diag(np.diag(S))
+        self.x.backward(dx)
+
+
+def phi(x):
+    return 0.5 * (np.tril(x) + np.tril(x, -1))
+
+
+def cholesky(x):
+    """
+    cholesky decomposition of positive-definite matrix
+    x = LL^T
+    Parameters
+    ----------
+    x : (d, d) tensor_like
+        positive-definite matrix
+    Returns
+    -------
+    L : (d, d)
+        cholesky decomposition
+    """
+    return Cholesky().forward(x)

books/PRML/PRML-master-Python/prml/nn/linalg/det.py

@@ -0,0 +1,35 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Determinant(Function):
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        self._equal_ndim(x, 2)
+        self.output = np.linalg.det(x.value)
+        if isinstance(self.x, Constant):
+            return Constant(self.output)
+        return Tensor(self.output, function=self)
+
+    def backward(self, delta):
+        dx = delta * self.output * np.linalg.inv(self.x.value.T)
+        self.x.backward(dx)
+
+
+def det(x):
+    """
+    determinant of a matrix
+    Parameters
+    ----------
+    x : (d, d) tensor_like
+        a matrix to compute its determinant
+    Returns
+    -------
+    output : (d, d) tensor_like
+        determinant of the input matrix
+    """
+    return Determinant().forward(x)

books/PRML/PRML-master-Python/prml/nn/linalg/inv.py

@@ -0,0 +1,35 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Inverse(Function):
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        self._equal_ndim(x, 2)
+        self.output = np.linalg.inv(x.value)
+        if isinstance(self.x, Constant):
+            return Constant(self.output)
+        return Tensor(self.output, function=self)
+
+    def backward(self, delta):
+        dx = -self.output.T @ delta @ self.output.T
+        self.x.backward(dx)
+
+
+def inv(x):
+    """
+    inverse of a matrix
+    Parameters
+    ----------
+    x : (d, d) tensor_like
+        a matrix to be inverted
+    Returns
+    -------
+    output : (d, d) tensor_like
+        inverse of the input
+    """
+    return Inverse().forward(x)

books/PRML/PRML-master-Python/prml/nn/linalg/logdet.py

@@ -0,0 +1,37 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class LogDeterminant(Function):
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        self._equal_ndim(x, 2)
+        sign, self.output = np.linalg.slogdet(x.value)
+        if sign != 1:
+            raise ValueError("matrix has to be positive-definite")
+        if isinstance(self.x, Constant):
+            return Constant(self.output)
+        return Tensor(self.output, function=self)
+
+    def backward(self, delta):
+        dx = delta * np.linalg.inv(self.x.value.T)
+        self.x.backward(dx)
+
+
+def logdet(x):
+    """
+    log determinant of a matrix
+    Parameters
+    ----------
+    x : (d, d) tensor_like
+        a matrix to compute its log determinant
+    Returns
+    -------
+    output : (d, d) tensor_like
+        determinant of the input matrix
+    """
+    return LogDeterminant().forward(x)

books/PRML/PRML-master-Python/prml/nn/linalg/solve.py

@@ -0,0 +1,43 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Solve(Function):
+
+    def forward(self, a, b):
+        a = self._convert2tensor(a)
+        b = self._convert2tensor(b)
+        self._equal_ndim(a, 2)
+        self._equal_ndim(b, 2)
+        self.a = a
+        self.b = b
+        self.output = np.linalg.solve(a.value, b.value)
+        if isinstance(self.a, Constant) and isinstance(self.b, Constant):
+            return Constant(self.output)
+        return Tensor(self.output, function=self)
+
+    def backward(self, delta):
+        db = np.linalg.solve(self.a.value.T, delta)
+        da = -db @ self.output.T
+        self.a.backward(da)
+        self.b.backward(db)
+
+
+def solve(a, b):
+    """
+    solve a linear matrix equation
+    ax = b
+    Parameters
+    ----------
+    a : (d, d) tensor_like
+        coefficient matrix
+    b : (d, k) tensor_like
+        dependent variable
+    Returns
+    -------
+    output : (d, k) tensor_like
+        solution of the equation
+    """
+    return Solve().forward(a, b)

books/PRML/PRML-master-Python/prml/nn/linalg/trace.py

@@ -0,0 +1,24 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Trace(Function):
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self._equal_ndim(x, 2)
+        self.x = x
+        output = np.trace(x.value)
+        if isinstance(self.x, Constant):
+            return Constant(output)
+        return Tensor(output, function=self)
+
+    def backward(self, delta):
+        dx = np.eye(self.x.shape[0], self.x.shape[1]) * delta
+        self.x.backward(dx)
+
+
+def trace(x):
+    return Trace().forward(x)

books/PRML/PRML-master-Python/prml/nn/math/__init__.py

@@ -0,0 +1,50 @@
+from prml.nn.math.add import add
+from prml.nn.math.divide import divide, rdivide
+from prml.nn.math.exp import exp
+from prml.nn.math.log import log
+from prml.nn.math.matmul import matmul, rmatmul
+from prml.nn.math.mean import mean
+from prml.nn.math.multiply import multiply
+from prml.nn.math.negative import negative
+from prml.nn.math.power import power, rpower
+from prml.nn.math.product import prod
+from prml.nn.math.sqrt import sqrt
+from prml.nn.math.square import square
+from prml.nn.math.subtract import subtract, rsubtract
+from prml.nn.math.sum import sum
+
+
+from prml.nn.tensor.tensor import Tensor
+Tensor.__add__ = add
+Tensor.__radd__ = add
+Tensor.__truediv__ = divide
+Tensor.__rtruediv__ = rdivide
+Tensor.mean = mean
+Tensor.__matmul__ = matmul
+Tensor.__rmatmul__ = rmatmul
+Tensor.__mul__ = multiply
+Tensor.__rmul__ = multiply
+Tensor.__neg__ = negative
+Tensor.__pow__ = power
+Tensor.__rpow__ = rpower
+Tensor.prod = prod
+Tensor.__sub__ = subtract
+Tensor.__rsub__ = rsubtract
+Tensor.sum = sum
+
+
+__all__ = [
+    "add",
+    "divide",
+    "exp",
+    "log",
+    "matmul",
+    "mean",
+    "multiply",
+    "power",
+    "prod",
+    "sqrt",
+    "square",
+    "subtract",
+    "sum"
+]

books/PRML/PRML-master-Python/prml/nn/math/abs.py

@@ -0,0 +1,27 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Abs(Function):
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        self.output = np.abs(x.value)
+        if isinstance(x, Constant):
+            return Constant(self.output)
+        self.sign = np.sign(x.value)
+        return Tensor(self.output, function=self)
+
+    def backward(self, delta):
+        dx = self.sign * delta
+        self.x.backward(dx)
+
+
+def abs(x):
+    """
+    element-wise absolute function
+    """
+    return Abs().forward(x)

books/PRML/PRML-master-Python/prml/nn/math/add.py

@@ -0,0 +1,40 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+from prml.nn.array.broadcast import broadcast_to
+
+
+class Add(Function):
+    """
+    add arguments element-wise
+    """
+
+    def _check_input(self, x, y):
+        x = self._convert2tensor(x)
+        y = self._convert2tensor(y)
+        if x.shape != y.shape:
+            shape = np.broadcast(x.value, y.value).shape
+            if x.shape != shape:
+                x = broadcast_to(x, shape)
+            if y.shape != shape:
+                y = broadcast_to(y, shape)
+        return x, y
+
+    def forward(self, x, y):
+        x, y = self._check_input(x, y)
+        self.x = x
+        self.y = y
+        if isinstance(self.x, Constant) and isinstance(self.y, Constant):
+            return Constant(x.value + y.value)
+        return Tensor(x.value + y.value, function=self)
+
+    def backward(self, delta):
+        dx = delta
+        dy = delta
+        self.x.backward(dx)
+        self.y.backward(dy)
+
+
+def add(x, y):
+    return Add().forward(x, y)

books/PRML/PRML-master-Python/prml/nn/math/divide.py

@@ -0,0 +1,44 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+from prml.nn.array.broadcast import broadcast_to
+
+
+class Divide(Function):
+    """
+    divide arguments element-wise
+    """
+
+    def _check_input(self, x, y):
+        x = self._convert2tensor(x)
+        y = self._convert2tensor(y)
+        if x.shape != y.shape:
+            shape = np.broadcast(x.value, y.value).shape
+            if x.shape != shape:
+                x = broadcast_to(x, shape)
+            if y.shape != shape:
+                y = broadcast_to(y, shape)
+        return x, y
+
+    def forward(self, x, y):
+        x, y = self._check_input(x, y)
+        self.x = x
+        self.y = y
+        if isinstance(self.x, Constant) and isinstance(self.y, Constant):
+            return Constant(x.value / y.value)
+        return Tensor(x.value / y.value, function=self)
+
+    def backward(self, delta):
+        dx = delta / self.y.value
+        dy = - delta * self.x.value / self.y.value ** 2
+        self.x.backward(dx)
+        self.y.backward(dy)
+
+
+def divide(x, y):
+    return Divide().forward(x, y)
+
+
+def rdivide(x, y):
+    return Divide().forward(y, x)

books/PRML/PRML-master-Python/prml/nn/math/exp.py

@@ -0,0 +1,26 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Exp(Function):
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        self.output = np.exp(x.value)
+        if isinstance(self.x, Constant):
+            return Constant(self.output)
+        return Tensor(self.output, function=self)
+
+    def backward(self, delta):
+        dx = self.output * delta
+        self.x.backward(dx)
+
+
+def exp(x):
+    """
+    element-wise exponential function
+    """
+    return Exp().forward(x)

books/PRML/PRML-master-Python/prml/nn/math/gamma.py

@@ -0,0 +1,26 @@
+import scipy.special as sp
+from prml.nn.function import Function
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+
+
+class Gamma(Function):
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        self.output = sp.gamma(x.value)
+        if isinstance(x, Constant):
+            return Constant(self.output)
+        return Tensor(self.output, function=self)
+
+    def backward(self, delta):
+        dx = delta * self.output * sp.digamma(self.x.value)
+        self.x.backward(dx)
+
+
+def gamma(x):
+    """
+    element-wise gamma function
+    """
+    return Gamma().forward(x)

books/PRML/PRML-master-Python/prml/nn/math/log.py

@@ -0,0 +1,31 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Log(Function):
+    """
+    element-wise natural logarithm of the input
+    y = log(x)
+    """
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        output = np.log(self.x.value)
+        if isinstance(self.x, Constant):
+            return Constant(output)
+        return Tensor(output, function=self)
+
+    def backward(self, delta):
+        dx = delta / self.x.value
+        self.x.backward(dx)
+
+
+def log(x):
+    """
+    element-wise natural logarithm of the input
+    y = log(x)
+    """
+    return Log().forward(x)

books/PRML/PRML-master-Python/prml/nn/math/matmul.py

@@ -0,0 +1,43 @@
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class MatMul(Function):
+    """
+    Matrix multiplication function
+    """
+
+    def _check_input(self, x, y):
+        x = self._convert2tensor(x)
+        y = self._convert2tensor(y)
+        self._equal_ndim(x, 2)
+        self._equal_ndim(y, 2)
+        if x.shape[1] != y.shape[0]:
+            raise ValueError(
+                "shapes {} and {} not aligned: {} (dim 1) != {} (dim 0)"
+                .format(x.shape, y.shape, x.shape[1], y.shape[0])
+            )
+        return x, y
+
+    def forward(self, x, y):
+        x, y = self._check_input(x, y)
+        self.x = x
+        self.y = y
+        if isinstance(self.x, Constant) and isinstance(self.y, Constant):
+            return Constant(x.value @ y.value)
+        return Tensor(x.value @ y.value, function=self)
+
+    def backward(self, delta):
+        dx = delta @ self.y.value.T
+        dy = self.x.value.T @ delta
+        self.x.backward(dx)
+        self.y.backward(dy)
+
+
+def matmul(x, y):
+    return MatMul().forward(x, y)
+
+
+def rmatmul(x, y):
+    return MatMul().forward(y, x)

books/PRML/PRML-master-Python/prml/nn/math/mean.py

@@ -0,0 +1,21 @@
+from prml.nn.math.sum import sum
+
+
+def mean(x, axis=None, keepdims=False):
+    """
+    returns arithmetic mean of the elements along given axis
+    """
+    if axis is None:
+        return sum(x, axis=None, keepdims=keepdims) / x.size
+    elif isinstance(axis, int):
+        N = x.shape[axis]
+        return sum(x, axis=axis, keepdims=keepdims) / N
+    elif isinstance(axis, tuple):
+        N = 1
+        for ax in axis:
+            N *= x.shape[ax]
+        return sum(x, axis=axis, keepdims=keepdims) / N
+    else:
+        raise TypeError(
+            "Unsupported type for axis: {}".format(type(axis))
+        )

books/PRML/PRML-master-Python/prml/nn/math/multiply.py

@@ -0,0 +1,40 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+from prml.nn.array.broadcast import broadcast_to
+
+
+class Multiply(Function):
+    """
+    multiply arguments element-wise
+    """
+
+    def _check_input(self, x, y):
+        x = self._convert2tensor(x)
+        y = self._convert2tensor(y)
+        if x.shape != y.shape:
+            shape = np.broadcast(x.value, y.value).shape
+            if x.shape != shape:
+                x = broadcast_to(x, shape)
+            if y.shape != shape:
+                y = broadcast_to(y, shape)
+        return x, y
+
+    def forward(self, x, y):
+        x, y = self._check_input(x, y)
+        self.x = x
+        self.y = y
+        if isinstance(self.x, Constant) and isinstance(self.y, Constant):
+            return Constant(x.value * y.value)
+        return Tensor(x.value * y.value, function=self)
+
+    def backward(self, delta):
+        dx = self.y.value * delta
+        dy = self.x.value * delta
+        self.x.backward(dx)
+        self.y.backward(dy)
+
+
+def multiply(x, y):
+    return Multiply().forward(x, y)

books/PRML/PRML-master-Python/prml/nn/math/negative.py

@@ -0,0 +1,28 @@
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Negative(Function):
+    """
+    element-wise negative
+    y = -x
+    """
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        if isinstance(self.x, Constant):
+            return Constant(-x.value)
+        return Tensor(-x.value, function=self)
+
+    def backward(self, delta):
+        dx = -delta
+        self.x.backward(dx)
+
+
+def negative(x):
+    """
+    element-wise negative
+    """
+    return Negative().forward(x)

books/PRML/PRML-master-Python/prml/nn/math/power.py

@@ -0,0 +1,57 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+from prml.nn.array.broadcast import broadcast_to
+
+
+class Power(Function):
+    """
+    First array elements raised to powers from second array
+    """
+
+    def _check_input(self, x, y):
+        x = self._convert2tensor(x)
+        y = self._convert2tensor(y)
+        if x.shape != y.shape:
+            shape = np.broadcast(x.value, y.value).shape
+            if x.shape != shape:
+                x = broadcast_to(x, shape)
+            if y.shape != shape:
+                y = broadcast_to(y, shape)
+        return x, y
+
+    def forward(self, x, y):
+        x, y = self._check_input(x, y)
+        self.x = x
+        self.y = y
+        self.output = np.power(x.value, y.value)
+        if isinstance(self.x, Constant) and isinstance(self.y, Constant):
+            return Constant(self.output)
+        return Tensor(self.output, function=self)
+
+    def backward(self, delta):
+        dx = self.y.value * np.power(self.x.value, self.y.value - 1) * delta
+        if self.x.size == 1:
+            if self.x.value > 0:
+                dy = self.output * np.log(self.x.value) * delta
+            else:
+                dy = None
+        else:
+            if (self.x.value > 0).all():
+                dy = self.output * np.log(self.x.value) * delta
+            else:
+                dy = None
+        self.x.backward(dx)
+        self.y.backward(dy)
+
+
+def power(x, y):
+    """
+    First array elements raised to powers from second array
+    """
+    return Power().forward(x, y)
+
+
+def rpower(x, y):
+    return Power().forward(y, x)

books/PRML/PRML-master-Python/prml/nn/math/product.py

@@ -0,0 +1,55 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Product(Function):
+
+    def __init__(self, axis=None, keepdims=False):
+        if isinstance(axis, int):
+            axis = (axis,)
+        elif isinstance(axis, tuple):
+            axis = tuple(sorted(axis))
+        self.axis = axis
+        self.keepdims = keepdims
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        self.output = np.prod(self.x.value, axis=self.axis, keepdims=True)
+        if not self.keepdims:
+            output = np.squeeze(self.output)
+            if output.size == 1:
+                output = output.item()
+        else:
+            output = self.output
+        if isinstance(self.x, Constant):
+            return Constant(output)
+        return Tensor(output, function=self)
+
+    def backward(self, delta):
+        if not self.keepdims and self.axis is not None:
+            for ax in self.axis:
+                delta = np.expand_dims(delta, ax)
+        dx = delta * self.output / self.x.value
+        self.x.backward(dx)
+
+
+def prod(x, axis=None, keepdims=False):
+    """
+    product of all element in the array
+    Parameters
+    ----------
+    x : tensor_like
+        input array
+    axis : int, tuple of ints
+        axis or axes along which a product is performed
+    keepdims : bool
+        keep dimensionality or not
+    Returns
+    -------
+    product : tensor_like
+        product of all element
+    """
+    return Product(axis=axis, keepdims=keepdims).forward(x)

books/PRML/PRML-master-Python/prml/nn/math/sqrt.py

@@ -0,0 +1,31 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Sqrt(Function):
+    """
+    element-wise square root of the input
+    y = sqrt(x)
+    """
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        self.output = np.sqrt(x.value)
+        if isinstance(self.x, Constant):
+            return Constant(self.output)
+        return Tensor(self.output, function=self)
+
+    def backward(self, delta):
+        dx = 0.5 * delta / self.output
+        self.x.backward(dx)
+
+
+def sqrt(x):
+    """
+    element-wise square root of the input
+    y = sqrt(x)
+    """
+    return Sqrt().forward(x)

books/PRML/PRML-master-Python/prml/nn/math/square.py

@@ -0,0 +1,30 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Square(Function):
+    """
+    element-wise square of the input
+    y = x * x
+    """
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        if isinstance(self.x, Constant):
+            return Constant(np.square(x.value))
+        return Tensor(np.square(x.value), function=self)
+
+    def backward(self, delta):
+        dx = 2 * self.x.value * delta
+        self.x.backward(dx)
+
+
+def square(x):
+    """
+    element-wise square of the input
+    y = x * x
+    """
+    return Square().forward(x)

books/PRML/PRML-master-Python/prml/nn/math/subtract.py

@@ -0,0 +1,44 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+from prml.nn.array.broadcast import broadcast_to
+
+
+class Subtract(Function):
+    """
+    subtract arguments element-wise
+    """
+
+    def _check_input(self, x, y):
+        x = self._convert2tensor(x)
+        y = self._convert2tensor(y)
+        if x.shape != y.shape:
+            shape = np.broadcast(x.value, y.value).shape
+            if x.shape != shape:
+                x = broadcast_to(x, shape)
+            if y.shape != shape:
+                y = broadcast_to(y, shape)
+        return x, y
+
+    def forward(self, x, y):
+        x, y = self._check_input(x, y)
+        self.x = x
+        self.y = y
+        if isinstance(self.x, Constant) and isinstance(self.y, Constant):
+            return Constant(x.value - y.value)
+        return Tensor(x.value - y.value, function=self)
+
+    def backward(self, delta):
+        dx = delta
+        dy = -delta
+        self.x.backward(dx)
+        self.y.backward(dy)
+
+
+def subtract(x, y):
+    return Subtract().forward(x, y)
+
+
+def rsubtract(x, y):
+    return Subtract().forward(y, x)

books/PRML/PRML-master-Python/prml/nn/math/sum.py

@@ -0,0 +1,46 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Sum(Function):
+    """
+    summation along given axis
+    y = sum_i=1^N x_i
+    """
+
+    def __init__(self, axis=None, keepdims=False):
+        if isinstance(axis, int):
+            axis = (axis,)
+        self.axis = axis
+        self.keepdims = keepdims
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        output = x.value.sum(axis=self.axis, keepdims=self.keepdims)
+        if isinstance(self.x, Constant):
+            return Constant(output)
+        return Tensor(output, function=self)
+
+    def backward(self, delta):
+        if isinstance(delta, np.ndarray) and (not self.keepdims) and (self.axis is not None):
+            axis_positive = []
+            for axis in self.axis:
+                if axis < 0:
+                    axis_positive.append(self.x.ndim + axis)
+                else:
+                    axis_positive.append(axis)
+            for axis in sorted(axis_positive):
+                delta = np.expand_dims(delta, axis)
+        dx = np.broadcast_to(delta, self.x.shape)
+        self.x.backward(dx)
+
+
+def sum(x, axis=None, keepdims=False):
+    """
+    returns summation of the elements along given axis
+    y = sum_i=1^N x_i
+    """
+    return Sum(axis=axis, keepdims=keepdims).forward(x)

books/PRML/PRML-master-Python/prml/nn/network.py

@@ -0,0 +1,90 @@
+from prml.nn.random.random import RandomVariable
+from prml.nn.tensor.parameter import Parameter
+
+
+class Network(object):
+    """
+    a base class for network building
+
+    Parameters
+    ----------
+    kwargs : tensor_like
+        parameters to be optimized
+
+    Attributes
+    ----------
+    parameter : dict
+        dictionary of parameters to be optimized
+    random_variable : dict
+        dictionary of random varibles
+    """
+
+    def __init__(self, **kwargs):
+        self.random_variable = {}
+        self.parameter = {}
+        for key, value in kwargs.items():
+            if isinstance(value, Parameter):
+                self.parameter[key] = value
+            else:
+                try:
+                    value = Parameter(value)
+                except TypeError:
+                    raise TypeError(f"invalid type argument: {type(value)}")
+                self.parameter[key] = value
+            object.__setattr__(self, key, value)
+
+    def __setattr__(self, key, value):
+        if isinstance(value, RandomVariable):
+            self.random_variable[key] = value
+        object.__setattr__(self, key, value)
+
+    def clear(self):
+        """
+        clear gradient and constructed bayesian network
+        """
+        for p in self.parameter.values():
+            p.cleargrad()
+        self.random_variable = {}
+
+    def log_pdf(self, coef=1.):
+        """
+        compute logarithm of probabilty density function
+        Parameters
+        ----------
+        coef : float
+            coefficient to balance likelihood and prior
+            assuming mini-batch size / whole data size for mini-batch training
+        Returns
+        -------
+        logp : tensor_like
+            logarithm of probability density function
+        """
+        logp = 0
+        for rv in self.random_variable.values():
+            if rv.observed:
+                logp += rv.log_pdf().sum()
+            else:
+                logp += coef * rv.log_pdf().sum()
+        return logp
+
+    def elbo(self, coef=1.):
+        """
+        compute evidence lower bound of this model
+        ln p(output) >= elbo
+        Parameters
+        ----------
+        coef : float
+            coefficient to balance likelihood and prior
+            assuming mini-batch size / whole data size for mini-batch training
+        Returns
+        -------
+        evidence : tensor_like
+            evidence lower bound
+        """
+        evidence = 0
+        for rv in self.random_variable.values():
+            if rv.observed:
+                evidence += rv.log_pdf().sum()
+            else:
+                evidence += -coef * rv.KLqp().sum()
+        return evidence

books/PRML/PRML-master-Python/prml/nn/nonlinear/log_softmax.py

@@ -0,0 +1,32 @@
+import numpy as np
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class LogSoftmax(Function):
+
+    def __init__(self, axis=-1):
+        self.axis = axis
+
+    def _logsumexp(self, x):
+        x_max = np.max(x, axis=self.axis, keepdims=True)
+        y = x - x_max
+        np.exp(y, out=y)
+        np.log(y.sum(axis=self.axis, keepdims=True), out=y)
+        y += x_max
+        return y
+
+    def _forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        self.output = x.value - self._logsumexp(x.value)
+        return Tensor(self.output, function=self)
+
+    def _backward(self, delta):
+        dx = delta
+        dx -= np.exp(self.output) * dx.sum(axis=self.axis, keepdims=True)
+        self.x.backward(dx)
+
+
+def log_softmax(x, axis=-1):
+    return LogSoftmax(axis=axis).forward(x)

books/PRML/PRML-master-Python/prml/nn/nonlinear/relu.py

@@ -0,0 +1,32 @@
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class ReLU(Function):
+    """
+    Rectified Linear Unit
+
+    y = max(x, 0)
+    """
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        output = x.value.clip(min=0)
+        if isinstance(x, Constant):
+            return Constant(output)
+        return Tensor(output, function=self)
+
+    def backward(self, delta):
+        dx = (self.x.value > 0) * delta
+        self.x.backward(dx)
+
+
+def relu(x):
+    """
+    Rectified Linear Unit
+
+    y = max(x, 0)
+    """
+    return ReLU().forward(x)

books/PRML/PRML-master-Python/prml/nn/nonlinear/sigmoid.py

@@ -0,0 +1,31 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Sigmoid(Function):
+    """
+    logistic sigmoid function
+    y = 1 / (1 + exp(-x))
+    """
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        self.output = np.tanh(x.value * 0.5) * 0.5 + 0.5
+        if isinstance(self.x, Constant):
+            return Constant(self.output)
+        return Tensor(self.output, function=self)
+
+    def backward(self, delta):
+        dx = self.output * (1 - self.output) * delta
+        self.x.backward(dx)
+
+
+def sigmoid(x):
+    """
+    logistic sigmoid function
+    y = 1 / (1 + exp(-x))
+    """
+    return Sigmoid().forward(x)

books/PRML/PRML-master-Python/prml/nn/nonlinear/softmax.py

@@ -0,0 +1,39 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Softmax(Function):
+
+    def __init__(self, axis=-1):
+        if not isinstance(axis, int):
+            raise TypeError("axis must be int")
+        self.axis = axis
+
+    def _softmax(self, array):
+        y = array - np.max(array, self.axis, keepdims=True)
+        np.exp(y, out=y)
+        y /= y.sum(self.axis, keepdims=True)
+        return y
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        self.output = self._softmax(x.value)
+        if isinstance(x, Constant):
+            return Constant(self.output)
+        return Tensor(self.output, function=self)
+
+    def backward(self, delta):
+        dx = self.output * delta
+        dx -= self.output * dx.sum(self.axis, keepdims=True)
+        self.x.backward(dx)
+
+
+def softmax(x, axis=-1):
+    """
+    softmax function along specified axis
+    y_k = exp(x_k) / sum_i(exp(x_i))
+    """
+    return Softmax(axis=axis).forward(x)

books/PRML/PRML-master-Python/prml/nn/nonlinear/softplus.py

@@ -0,0 +1,27 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Softplus(Function):
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        output = np.maximum(x.value, 0) + np.log1p(np.exp(-np.abs(x.value)))
+        if isinstance(x, Constant):
+            return Constant(output)
+        return Tensor(output, function=self)
+
+    def backward(self, delta):
+        dx = (np.tanh(0.5 * self.x.value) * 0.5 + 0.5) * delta
+        self.x.backward(dx)
+
+
+def softplus(x):
+    """
+    smoothed rectified linear unit
+    log(1 + exp(x))
+    """
+    return Softplus().forward(x)

books/PRML/PRML-master-Python/prml/nn/nonlinear/tanh.py

@@ -0,0 +1,27 @@
+import numpy as np
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+from prml.nn.function import Function
+
+
+class Tanh(Function):
+
+    def forward(self, x):
+        x = self._convert2tensor(x)
+        self.x = x
+        self.output = np.tanh(x.value)
+        if isinstance(self.x, Constant):
+            return Constant(self.output)
+        return Tensor(self.output, function=self)
+
+    def backward(self, delta):
+        dx = (1 - np.square(self.output)) * delta
+        self.x.backward(dx)
+
+
+def tanh(x):
+    """
+    hyperbolic tangent function
+    y = (exp(x) - exp(-x)) / (exp(x) + exp(-x))
+    """
+    return Tanh().forward(x)

books/PRML/PRML-master-Python/prml/nn/optimizer/__init__.py

@@ -0,0 +1,16 @@
+from prml.nn.optimizer.ada_delta import AdaDelta
+from prml.nn.optimizer.ada_grad import AdaGrad
+from prml.nn.optimizer.adam import Adam
+from prml.nn.optimizer.gradient_ascent import GradientAscent
+from prml.nn.optimizer.momentum import Momentum
+from prml.nn.optimizer.rmsprop import RMSProp
+
+
+__all__ = [
+    "AdaDelta",
+    "AdaGrad",
+    "Adam",
+    "GradientAscent",
+    "Momentum",
+    "RMSProp"
+]

books/PRML/PRML-master-Python/prml/nn/optimizer/ada_delta.py

@@ -0,0 +1,31 @@
+import numpy as np
+from prml.nn.optimizer.optimizer import Optimizer
+
+
+class AdaDelta(Optimizer):
+    """
+    AdaDelta optimizer
+    """
+
+    def __init__(self, parameter, rho=0.95, epsilon=1e-8):
+        super().__init__(parameter, None)
+        self.rho = rho
+        self.epsilon = epsilon
+        self.mean_squared_deriv = []
+        self.mean_squared_update = []
+        for p in self.parameter:
+            self.mean_squared_deriv.append(np.zeros(p.shape))
+            self.mean_squared_update.append(np.zeros(p.shape))
+
+    def update(self):
+        self.increment_iteration()
+        for p, msd, msu in zip(self.parameter, self.mean_squared_deriv, self.mean_squared_update):
+            if p.grad is None:
+                continue
+            grad = p.grad
+            msd *= self.rho
+            msd += (1 - self.rho) * grad ** 2
+            delta = np.sqrt((msu + self.epsilon) / (msd + self.epsilon)) * grad
+            msu *= self.rho
+            msu *= (1 - self.rho) * delta ** 2
+            p.value += delta

books/PRML/PRML-master-Python/prml/nn/optimizer/ada_grad.py

@@ -0,0 +1,32 @@
+import numpy as np
+from prml.nn.optimizer.optimizer import Optimizer
+
+
+class AdaGrad(Optimizer):
+    """
+    AdaGrad optimizer
+    initialization
+    G = 0
+    update rule
+    G += gradient ** 2
+    param -= learning_rate * gradient / sqrt(G + eps)
+    """
+
+    def __init__(self, parameter, learning_rate=0.001, epsilon=1e-8):
+        super().__init__(parameter, learning_rate)
+        self.epsilon = epsilon
+        self.G = []
+        for p in self.parameter:
+            self.G.append(np.zeros(p.shape))
+
+    def update(self):
+        """
+        update parameters
+        """
+        self.increment_iteration()
+        for p, G in zip(self.parameter, self.G):
+            if p.grad is None:
+                continue
+            grad = p.grad
+            G += grad ** 2
+            p.value += self.learning_rate * grad / (np.sqrt(G) + self.epsilon)

books/PRML/PRML-master-Python/prml/nn/optimizer/adam.py

@@ -0,0 +1,67 @@
+import numpy as np
+from prml.nn.optimizer.optimizer import Optimizer
+
+
+class Adam(Optimizer):
+    """
+    Adam optimizer
+    initialization
+    m1 = 0 (Initial 1st moment of gradient)
+    m2 = 0 (Initial 2nd moment of gradient)
+    n_iter = 0
+    update rule
+    n_iter += 1
+    learning_rate *= sqrt(1 - beta2^n) / (1 - beta1^n)
+    m1 = beta1 * m1 + (1 - beta1) * gradient
+    m2 = beta2 * m2 + (1 - beta2) * gradient^2
+    param += learning_rate * m1 / (sqrt(m2) + epsilon)
+    """
+
+    def __init__(self, parameter, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8):
+        """
+        construct Adam optimizer
+        Parameters
+        ----------
+        parameters : list
+            list of parameters to be optimized
+        learning_rate : float
+        beta1 : float
+            exponential decay rate for the 1st moment
+        beta2 : float
+            exponential decay rate for the 2nd moment
+        epsilon : float
+            small constant to be added to denominator for numerical stability
+        Attributes
+        ----------
+        n_iter : int
+            number of iterations performed
+        moment1 : dict
+            1st moment of each learnable parameter
+        moment2 : dict
+            2nd moment of each learnable parameter
+        """
+        super().__init__(parameter, learning_rate)
+        self.beta1 = beta1
+        self.beta2 = beta2
+        self.epsilon = epsilon
+        self.moment1 = []
+        self.moment2 = []
+        for p in self.parameter:
+            self.moment1.append(np.zeros(p.shape))
+            self.moment2.append(np.zeros(p.shape))
+
+    def update(self):
+        """
+        update parameter of the neural network
+        """
+        self.increment_iteration()
+        lr = (
+            self.learning_rate
+            * (1 - self.beta2 ** self.n_iter) ** 0.5
+            / (1 - self.beta1 ** self.n_iter))
+        for p, m1, m2 in zip(self.parameter, self.moment1, self.moment2):
+            if p.grad is None:
+                continue
+            m1 += (1 - self.beta1) * (p.grad - m1)
+            m2 += (1 - self.beta2) * (p.grad ** 2 - m2)
+            p.value += lr * m1 / (np.sqrt(m2) + self.epsilon)

books/PRML/PRML-master-Python/prml/nn/optimizer/eve.py

@@ -0,0 +1,101 @@
+import numpy as np
+from prml.nn.optimizer.optimizer import Optimizer
+
+
+class Eve(Optimizer):
+    """
+    Eve optimizer
+
+    initialization
+    m1 = 0 (initial 1st moment of gradient)
+    m2 = 0 (initial 2nd moment of gradient)
+    n_iter = 0
+
+    update rule
+    n_iter += 1
+    learning
+    """
+
+    def __init__(self,
+                 network,
+                 learning_rate=0.001,
+                 beta1=0.9,
+                 beta2=0.999,
+                 beta3=0.999,
+                 lower_threshold=0.1,
+                 upper_threshold=10.,
+                 epsilon=1e-8):
+        """
+        construct Eve optimizer
+
+        Parameters
+        ----------
+        network : Network
+            neural network to be optmized
+        learning_rate : float
+        beta1 : float
+            exponential decay rate for the 1st moment
+        beta2 : float
+            exponential decay rate for the 2nd moment
+        beta3 : float
+            exponential decay rate for computing relative change
+        lower_threshold : float
+             lower threshold for relative change
+        upper_threshold : float
+             upper threshold for relative change
+        epsilon : float
+            small constant to be added to denominator for numerical stability
+
+        Attributes
+        ----------
+        n_iter : int
+            number of iterations performed
+        moment1 : dict
+            1st moment of each parameter
+        moment2 : dict
+            2nd moment of each parameter
+        """
+        super().__init__(network, learning_rate)
+        self.beta1 = beta1
+        self.beta2 = beta2
+        self.beta3 = beta3
+        self.lower_threshold = lower_threshold
+        self.upper_threshold = upper_threshold
+        self.epsilon = epsilon
+        self.moment1 = {}
+        self.moment2 = {}
+        self.f = 1.
+        self.d = 1.
+        for key, param in self.params.items():
+            self.moment1[key] = np.zeros(param.shape)
+            self.moment2[key] = np.zeros(param.shape)
+
+    def update(self, loss):
+        loss = float(loss)
+        self.increment_iteration()
+        if self.n_iter > 1:
+            if loss > self.f:
+                delta = self.lower_threshold + 1
+                Delta = self.upper_threshold + 1
+            else:
+                delta = 1 / (self.upper_threshold + 1)
+                Delta = 1 / (self.lower_threshold + 1)
+            c = min(max(delta, loss / self.f), Delta)
+            f = c * self.f
+            r = abs(f - self.f) / min(f, self.f)
+            self.d = self.beta3 * self.d * (1 - self.beta3) * r
+            self.f = f
+        else:
+            self.f = loss
+            self.d = 1
+        lr = (
+            self.learning_rate
+            * (1 - self.beta2 ** self.n_iter) ** 0.5
+            / (1 - self.beta1 ** self.n_iter)
+        )
+        for key, param in self.params.items():
+            m1 = self.moment1[key]
+            m2 = self.moment2[key]
+            m1 += (1 - self.beta1) * (param.grad - m1)
+            m2 += (1 - self.beta2) * (param.grad ** 2 - m2)
+            param.value -= lr * m1 / (self.d * np.sqrt(m2) + self.epsilon)

books/PRML/PRML-master-Python/prml/nn/optimizer/gradient_ascent.py

@@ -0,0 +1,18 @@
+from prml.nn.optimizer.optimizer import Optimizer
+
+
+class GradientAscent(Optimizer):
+    """
+    gradient ascent optimizer
+    parameter += learning_rate * gradient
+    """
+
+    def update(self):
+        """
+        update parameters to be optimized
+        """
+        self.increment_iteration()
+        for p in self.parameter:
+            if p.grad is None:
+                continue
+            p.value += self.learning_rate * p.grad

books/PRML/PRML-master-Python/prml/nn/optimizer/momentum.py

@@ -0,0 +1,29 @@
+import numpy as np
+from prml.nn.optimizer.optimizer import Optimizer
+
+
+class Momentum(Optimizer):
+    """
+    Momentum optimizer
+    initialization
+    v = 0
+    update rule
+    v = v * momentum - learning_rate * gradient
+    param += v
+    """
+
+    def __init__(self, parameter, learning_rate, momentum=0.9):
+        super().__init__(parameter, learning_rate)
+        self.momentum = momentum
+        self.inertia = []
+        for p in self.parameter:
+            self.inertia.append(np.zeros(p.shape))
+
+    def update(self):
+        self.increment_iteration()
+        for p, inertia in zip(self.parameter, self.inertia):
+            if p.grad is None:
+                continue
+            inertia *= self.momentum
+            inertia -= self.learning_rate * p.grad
+            p.value += inertia

books/PRML/PRML-master-Python/prml/nn/optimizer/optimizer.py

@@ -0,0 +1,52 @@
+from prml.nn.network import Network
+
+
+class Optimizer(object):
+    """
+    Optimizer to train neural network
+    """
+
+    def __init__(self, parameter, learning_rate):
+        """
+        construct optimizer
+        Parameters
+        ----------
+        parameter : list, dict, Network
+            list of parameter to be optimized
+        learning_rate : float
+            update rate of parameter to be optimized
+        Attributes
+        ----------
+        n_iter : int
+            number of iterations performed
+        """
+        if isinstance(parameter, Network):
+            parameter = parameter.parameter
+        if isinstance(parameter, dict):
+            parameter = list(parameter.values())
+        self.parameter = parameter
+        self.learning_rate = learning_rate
+        self.n_iter = 0
+
+    def cleargrad(self):
+        for p in self.parameter:
+            p.cleargrad()
+
+    def set_decay(self, decay_rate, decay_step):
+        """
+        set exponential decay parameters
+        Parameters
+        ----------
+        decay_rate : float
+            dacay rate of the learning rate
+        decay_step : int
+            steps to decay the learning rate
+        """
+        self.decay_rate = decay_rate
+        self.decay_step = decay_step
+
+    def increment_iteration(self):
+        self.n_iter += 1
+        if hasattr(self, "decay_rate"):
+            if self.n_iter % self.decay_step == 0:
+                self.learning_rate *= self.decay_rate

books/PRML/PRML-master-Python/prml/nn/optimizer/rmsprop.py

@@ -0,0 +1,36 @@
+import numpy as np
+from prml.nn.optimizer.optimizer import Optimizer
+
+
+class RMSProp(Optimizer):
+    """
+    RMSProp optimizer
+    initial
+    msg = 0
+    update rule
+    msg = rho * msg + (1 - rho) * gradient ** 2
+    param -= learning_rate * gradient / (sqrt(msg) + eps)
+    """
+
+    def __init__(self, parameter, learning_rate=1e-3, rho=0.9, epsilon=1e-8):
+        super().__init__(parameter, learning_rate)
+        self.rho = rho
+        self.epsilon = epsilon
+        self.mean_squared_grad = []
+        for p in self.parameter:
+            self.mean_squared_grad.append(np.zeros(p.shape))
+
+    def update(self):
+        """
+        update parameters
+        """
+        self.increment_iteration()
+        for p, msg in zip(self.parameter, self.mean_squared_grad):
+            if p.grad is None:
+                continue
+            grad = p.grad
+            msg *= self.rho
+            msg += (1 - self.rho) * grad ** 2
+            p.value -= (
+                self.learning_rate * grad / (np.sqrt(msg) + self.epsilon)
+            )

books/PRML/PRML-master-Python/prml/nn/random/__init__.py

@@ -0,0 +1,25 @@
+from prml.nn.random.bernoulli import Bernoulli
+from prml.nn.random.categorical import Categorical
+from prml.nn.random.cauchy import Cauchy
+from prml.nn.random.dirichlet import Dirichlet
+from prml.nn.random.dropout import dropout
+from prml.nn.random.exponential import Exponential
+from prml.nn.random.gamma import Gamma
+from prml.nn.random.gaussian import Gaussian
+from prml.nn.random.gaussian_mixture import GaussianMixture
+from prml.nn.random.laplace import Laplace
+from prml.nn.random.multivariate_gaussian import MultivariateGaussian
+
+
+__all__ = [
+    "Bernoulli",
+    "Categorical",
+    "Cauchy",
+    "Dirichlet",
+    "Exponential",
+    "Gamma",
+    "Gaussian",
+    "GaussianMixture",
+    "Laplace",
+    "MultivariateGaussian"
+]

books/PRML/PRML-master-Python/prml/nn/random/bernoulli.py

@@ -0,0 +1,126 @@
+import numpy as np
+from prml.nn.array.broadcast import broadcast_to
+from prml.nn.function import Function
+from prml.nn.math.log import log
+from prml.nn.nonlinear.sigmoid import sigmoid
+from prml.nn.random.random import RandomVariable
+from prml.nn.tensor.tensor import Tensor
+
+
+class Bernoulli(RandomVariable):
+    """
+    Bernoulli distribution
+    p(x|mu) = mu^x (1 - mu)^(1 - x)
+    Parameters
+    ----------
+    mu : tensor_like
+        probability of value 1
+    logit : tensor_like
+        log-odd of value 1
+    data : tensor_like
+        observed data
+    p : RandomVariable
+        original distribution of a model
+    """
+
+    def __init__(self, mu=None, logit=None, data=None, p=None):
+        super().__init__(data, p)
+        if mu is not None and logit is None:
+            mu = self._convert2tensor(mu)
+            self.mu = mu
+        elif mu is None and logit is not None:
+            logit = self._convert2tensor(logit)
+            self.logit = logit
+        elif mu is None and logit is None:
+            raise ValueError("Either mu or logit must not be None")
+        else:
+            raise ValueError("Cannot assign both mu and logit")
+
+    @property
+    def mu(self):
+        try:
+            return self.parameter["mu"]
+        except KeyError:
+            return sigmoid(self.logit)
+
+    @mu.setter
+    def mu(self, mu):
+        try:
+            inrange = (0 <= mu.value <= 1)
+        except ValueError:
+            inrange = ((mu.value >= 0).all() and (mu.value <= 1).all())
+
+        if not inrange:
+            raise ValueError("value of mu must all be positive")
+        self.parameter["mu"] = mu
+
+    @property
+    def logit(self):
+        try:
+            return self.parameter["logit"]
+        except KeyError:
+            raise AttributeError("no attribute named logit")
+
+    @logit.setter
+    def logit(self, logit):
+        self.parameter["logit"] = logit
+
+    def forward(self):
+        return (np.random.uniform(size=self.mu.shape) < self.mu.value).astype(np.int)
+
+    def _pdf(self, x):
+        return self.mu ** x * (1 - self.mu) ** (1 - x)
+
+    def _log_pdf(self, x):
+        try:
+            return -SigmoidCrossEntropy().forward(self.logit, x)
+        except AttributeError:
+            return x * log(self.mu) + (1 - x) * log(1 - self.mu)
+
+
+class SigmoidCrossEntropy(Function):
+    """
+    sum of cross entropies for binary data
+    logistic sigmoid
+    y_i = 1 / (1 + exp(-x_i))
+    cross_entropy_i = -t_i * log(y_i) - (1 - t_i) * log(1 - y_i)
+    Parameters
+    ----------
+    x : ndarary
+        input logit
+    y : ndarray
+        corresponding target binaries
+    """
+
+    def _check_input(self, x, t):
+        x = self._convert2tensor(x)
+        t = self._convert2tensor(t)
+        if x.shape != t.shape:
+            shape = np.broadcast(x.value, t.value).shape
+            if x.shape != shape:
+                x = broadcast_to(x, shape)
+            if t.shape != shape:
+                t = broadcast_to(t, shape)
+        return x, t
+
+
+    def forward(self, x, t):
+        x, t = self._check_input(x, t)
+        self.x = x
+        self.t = t
+        # y = self.forward(x)
+        # np.clip(y, 1e-10, 1 - 1e-10, out=y)
+        # return np.sum(-t * np.log(y) - (1 - t) * np.log(1 - y))
+        loss = (
+            np.maximum(x.value, 0)
+            - t.value * x.value
+            + np.log1p(np.exp(-np.abs(x.value)))
+        )
+        return Tensor(loss, function=self)
+
+    def backward(self, delta):
+        y = np.tanh(self.x.value * 0.5) * 0.5 + 0.5
+        dx = delta * (y - self.t.value)
+        dt = - delta * self.x.value
+        self.x.backward(dx)
+        self.t.backward(dt)

books/PRML/PRML-master-Python/prml/nn/random/categorical.py

@@ -0,0 +1,131 @@
+import numpy as np
+from prml.nn.array.broadcast import broadcast_to
+from prml.nn.function import Function
+from prml.nn.math.log import log
+from prml.nn.math.product import prod
+from prml.nn.nonlinear.softmax import softmax
+from prml.nn.random.random import RandomVariable
+from prml.nn.tensor.tensor import Tensor
+
+
+class Categorical(RandomVariable):
+    """
+    Categorical distribution
+    Parameters
+    ----------
+    mu : (..., K) tensor_like
+        probability of each index
+    logit : (..., K) tensor_like
+        log-odd of each index
+    axis : int
+        axis along which represents each outcome
+    data : tensor_like
+        realization
+    p : RandomVariable
+        original distribution of a model
+    Attributes
+    ----------
+    n_category : int
+        number of categories
+    """
+
+    def __init__(self, mu=None, logit=None, axis=-1, data=None, p=None):
+        super().__init__(data, p)
+        assert axis == -1
+        self.axis = axis
+        if mu is not None and logit is None:
+            self.mu = self._convert2tensor(mu)
+        elif mu is None and logit is not None:
+            self.logit = self._convert2tensor(logit)
+        elif mu is None and logit is None:
+            raise ValueError("Either mu or logit must not be None")
+        else:
+            raise ValueError("Cannot assign both mu and logit")
+
+    @property
+    def mu(self):
+        try:
+            return self.parameter["mu"]
+        except KeyError:
+            return softmax(self.parameter["logit"])
+
+    @mu.setter
+    def mu(self, mu):
+        self._atleast_ndim(mu, 1)
+        if not ((mu.value >= 0).all() and (mu.value <= 1).all()):
+            raise ValueError("values of mu must be in [0, 1]")
+        if not np.allclose(mu.value.sum(axis=self.axis), 1):
+            raise ValueError(f"mu must be normalized along axis {self.axis}")
+        self.parameter["mu"] = mu
+        self.n_category = mu.shape[self.axis]
+
+    @property
+    def logit(self):
+        try:
+            return self.parameter["logit"]
+        except KeyError:
+            raise AttributeError("no attribute named logit")
+
+    @logit.setter
+    def logit(self, logit):
+        self._atleast_ndim(logit, 1)
+        self.parameter["logit"] = logit
+        self.n_category = logit.shape[self.axis]
+
+    def forward(self):
+        if self.mu.ndim == 1:
+            index = np.random.choice(self.n_category, p=self.mu.value)
+            return np.eye(self.n_category)[index]
+        elif self.mu.ndim == 2:
+            indices = np.array(
+                [np.random.choice(self.n_category, p=p.value) for p in self.mu.value]
+            )
+            return np.eye(self.n_category)[indices]
+        else:
+            raise NotImplementedError
+
+    def _pdf(self, x):
+        return prod(self.mu ** x, axis=self.axis)
+
+    def _log_pdf(self, x):
+        try:
+            return -SoftmaxCrossEntropy(axis=self.axis).forward(self.logit, x)
+        except (KeyError, AttributeError):
+            return (x * log(self.mu)).sum(axis=self.axis)
+
+
+class SoftmaxCrossEntropy(Function):
+
+    def __init__(self, axis=-1):
+        self.axis = axis
+
+    def _check_input(self, x, t):
+        x = self._convert2tensor(x)
+        t = self._convert2tensor(t)
+        if x.shape != t.shape:
+            shape = np.broadcast(x.value, t.value).shape
+            if x.shape != shape:
+                x = broadcast_to(x, shape)
+            if t.shape != shape:
+                t = broadcast_to(t, shape)
+        return x, t
+
+    def _softmax(self, array):
+        y = np.exp(array - np.max(array, self.axis, keepdims=True))
+        y /= np.sum(y, self.axis, keepdims=True)
+        return y
+
+    def forward(self, x, t):
+        x, t = self._check_input(x, t)
+        self.x = x
+        self.t = t
+        self.y = self._softmax(x.value)
+        np.clip(self.y, 1e-10, 1, out=self.y)
+        loss = -t.value * np.log(self.y)
+        return Tensor(loss, function=self)
+
+    def backward(self, delta):
+        dx = delta * (self.y - self.t.value)
+        dt = - delta * np.log(self.y)
+        self.x.backward(dx)
+        self.t.backward(dt)

books/PRML/PRML-master-Python/prml/nn/random/cauchy.py

@@ -0,0 +1,88 @@
+import numpy as np
+from prml.nn.array.broadcast import broadcast_to
+from prml.nn.math.log import log
+from prml.nn.math.square import square
+from prml.nn.random.random import RandomVariable
+from prml.nn.tensor.constant import Constant
+from prml.nn.tensor.tensor import Tensor
+
+
+class Cauchy(RandomVariable):
+    """
+    Cauchy distribution aka Lorentz distribution
+    p(x|x0(loc), scale)
+    = 1 / [pi*scale * {1 + (x - x0)^2 / scale^2}]
+    Parameters
+    ----------
+    loc : tensor_like
+        location parameter
+    scale : tensor_like
+        scale parameter
+    data : tensor_like
+        realization
+    p : RandomVariable
+        original distribution of a model
+    """
+
+    def __init__(self, loc, scale, data=None, p=None):
+        super().__init__(data, p)
+        self.loc, self.scale = self._check_input(loc, scale)
+
+    def _check_input(self, loc, scale):
+        loc = self._convert2tensor(loc)
+        scale = self._convert2tensor(scale)
+        if loc.shape != scale.shape:
+            shape = np.broadcast(loc.value, scale.value).shape
+            if loc.shape != shape:
+                loc = broadcast_to(loc, shape)
+            if scale.shape != shape:
+                scale = broadcast_to(scale, shape)
+        return loc, scale
+
+    @property
+    def loc(self):
+        return self.parameter["loc"]
+
+    @loc.setter
+    def loc(self, loc):
+        self.parameter["loc"] = loc
+
+    @property
+    def scale(self):
+        return self.parameter["scale"]
+
+    @scale.setter
+    def scale(self, scale):
+        try:
+            ispositive = (scale.value > 0).all()
+        except AttributeError:
+            ispositive = (scale.value > 0)
+
+        if not ispositive:
+            raise ValueError("value of scale must be positive")
+        self.parameter["scale"] = scale
+
+    def forward(self):
+        self.eps = np.random.standard_cauchy(size=self.loc.shape)
+        self.output = self.scale.value * self.eps + self.loc.value
+        if isinstance(self.loc, Constant):
+            return Constant(self.output)
+        return Tensor(self.output, function=self)
+
+    def backward(self, delta):
+        dloc = delta
+        dscale = delta * self.eps
+        self.loc.backward(dloc)
+        self.scale.backward(dscale)
+
+    def _pdf(self, x):
+        return (
+            1 / (np.pi * self.scale * (1 + square((x - self.loc) / self.scale)))
+        )
+
+    def _log_pdf(self, x):
+        return (
+            -np.log(np.pi)
+            - log(self.scale)
+            - log(1 + square((x - self.loc) / self.scale))
+        )

books/PRML/PRML-master-Python/prml/nn/random/dirichlet.py

@@ -0,0 +1,72 @@
+import numpy as np
+from prml.nn.math.gamma import gamma
+from prml.nn.math.log import log
+from prml.nn.math.product import prod
+from prml.nn.math.sum import sum
+from prml.nn.random.random import RandomVariable
+from prml.nn.tensor.tensor import Tensor
+
+
+class Dirichlet(RandomVariable):
+    """
+    Dirichlet distribution
+    Parameters
+    ----------
+    alpha : (..., K) tensor_like
+        pseudo-count of each outcome
+    axis : int
+        axis along which represents each outcome
+    data : tensor_like
+        realization
+    p : RandomVariable
+        original distribution of a model
+    Attributes
+    ----------
+    n_category : int
+        number of categories
+    """
+
+    def __init__(self, alpha, axis=-1, data=None, p=None):
+        super().__init__(data, p)
+        assert axis == -1
+        self.axis = axis
+        self.alpha = self._convert2tensor(alpha)
+
+    @property
+    def alpha(self):
+        return self.parameter["alpha"]
+
+    @alpha.setter
+    def alpha(self, alpha):
+        self._atleast_ndim(alpha, 1)
+        if (alpha.value <= 0).any():
+            raise ValueError("alpha must all be positive")
+        self.parameter["alpha"] = alpha
+
+    def forward(self):
+        if self.alpha.ndim == 1:
+            return Tensor(np.random.dirichlet(self.alpha.value), function=self)
+        else:
+            raise NotImplementedError
+
+    def backward(self):
+        raise NotImplementedError
+
+    def _pdf(self, x):
+        return (
+            gamma(self.alpha.sum(axis=self.axis))
+            * prod(
+                x ** (self.alpha - 1)
+                / gamma(self.alpha),
+                axis=self.axis
+            )
+        )
+
+    def _log_pdf(self, x):
+        return (
+            log(gamma(self.alpha.sum(axis=self.axis)))
+            + sum(
+                (self.alpha - 1) * log(x) - log(gamma(self.alpha)),
+                axis=self.axis
+            )
+        )