wehub/rushter--mlalgorithms
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:39:55 +08:00
commit 7ee4420c10
87 changed files with 15222 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/__init__.py
+1
View File
@@ -0,0 +1 @@
# coding: utf-8
examples/gaussian_mixture.py
+46
View File
@@ -0,0 +1,46 @@
import random
import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets
from mla.kmeans import KMeans
from mla.gaussian_mixture import GaussianMixture
random.seed(1)
np.random.seed(6)
def make_clusters(skew=True, *arg, **kwargs):
X, y = datasets.make_blobs(*arg, **kwargs)
if skew:
nrow = X.shape[1]
for i in np.unique(y):
X[y == i] = X[y == i].dot(np.random.random((nrow, nrow)) - 0.5)
return X, y
def KMeans_and_GMM(K):
COLOR = "bgrcmyk"
X, y = make_clusters(skew=True, n_samples=1500, centers=K)
_, axes = plt.subplots(1, 3)
# Ground Truth
axes[0].scatter(X[:, 0], X[:, 1], c=[COLOR[int(assignment)] for assignment in y])
axes[0].set_title("Ground Truth")
# KMeans
kmeans = KMeans(K=K, init="++")
kmeans.fit(X)
kmeans.predict()
axes[1].set_title("KMeans")
kmeans.plot(ax=axes[1], holdon=True)
# Gaussian Mixture
gmm = GaussianMixture(K=K, init="kmeans")
gmm.fit(X)
axes[2].set_title("Gaussian Mixture")
gmm.plot(ax=axes[2])
if __name__ == "__main__":
KMeans_and_GMM(4)
examples/gbm.py
+70
View File
@@ -0,0 +1,70 @@
import logging
from sklearn.datasets import make_classification
from sklearn.datasets import make_regression
from sklearn.metrics import roc_auc_score
try:
from sklearn.model_selection import train_test_split
except ImportError:
from sklearn.cross_validation import train_test_split
from mla.ensemble.gbm import GradientBoostingClassifier, GradientBoostingRegressor
from mla.metrics.metrics import mean_squared_error
logging.basicConfig(level=logging.DEBUG)
def classification():
# Generate a random binary classification problem.
X, y = make_classification(
n_samples=350,
n_features=15,
n_informative=10,
random_state=1111,
n_classes=2,
class_sep=1.0,
n_redundant=0,
)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.15, random_state=1111
)
model = GradientBoostingClassifier(
n_estimators=50, max_depth=4, max_features=8, learning_rate=0.1
)
model.fit(X_train, y_train)
predictions = model.predict(X_test)
print(predictions)
print(predictions.min())
print(predictions.max())
print("classification, roc auc score: %s" % roc_auc_score(y_test, predictions))
def regression():
# Generate a random regression problem
X, y = make_regression(
n_samples=500,
n_features=5,
n_informative=5,
n_targets=1,
noise=0.05,
random_state=1111,
bias=0.5,
)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.1, random_state=1111
)
model = GradientBoostingRegressor(n_estimators=25, max_depth=5, max_features=3)
model.fit(X_train, y_train)
predictions = model.predict(X_test)
print(
"regression, mse: %s"
% mean_squared_error(y_test.flatten(), predictions.flatten())
)
if __name__ == "__main__":
classification()
# regression()
examples/kmeans.py
+21
View File
@@ -0,0 +1,21 @@
import numpy as np
from sklearn.datasets import make_blobs
from mla.kmeans import KMeans
def kmeans_example(plot=False):
X, y = make_blobs(
centers=4, n_samples=500, n_features=2, shuffle=True, random_state=42
)
clusters = len(np.unique(y))
k = KMeans(K=clusters, max_iters=150, init="++")
k.fit(X)
k.predict()
if plot:
k.plot()
if __name__ == "__main__":
kmeans_example(plot=True)
examples/linear_models.py
+60
View File
@@ -0,0 +1,60 @@
import logging
try:
from sklearn.model_selection import train_test_split
except ImportError:
from sklearn.cross_validation import train_test_split
from sklearn.datasets import make_classification
from sklearn.datasets import make_regression
from mla.linear_models import LinearRegression, LogisticRegression
from mla.metrics.metrics import mean_squared_error, accuracy
# Change to DEBUG to see convergence
logging.basicConfig(level=logging.ERROR)
def regression():
# Generate a random regression problem
X, y = make_regression(
n_samples=10000,
n_features=100,
n_informative=75,
n_targets=1,
noise=0.05,
random_state=1111,
bias=0.5,
)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.25, random_state=1111
)
model = LinearRegression(lr=0.01, max_iters=2000, penalty="l2", C=0.03)
model.fit(X_train, y_train)
predictions = model.predict(X_test)
print("regression mse", mean_squared_error(y_test, predictions))
def classification():
# Generate a random binary classification problem.
X, y = make_classification(
n_samples=1000,
n_features=100,
n_informative=75,
random_state=1111,
n_classes=2,
class_sep=2.5,
)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.1, random_state=1111
)
model = LogisticRegression(lr=0.01, max_iters=500, penalty="l1", C=0.01)
model.fit(X_train, y_train)
predictions = model.predict(X_test)
print("classification accuracy", accuracy(y_test, predictions))
if __name__ == "__main__":
regression()
classification()
examples/naive_bayes.py
+31
View File
@@ -0,0 +1,31 @@
from sklearn.datasets import make_classification
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
from mla.naive_bayes import NaiveBayesClassifier
def classification():
# Generate a random binary classification problem.
X, y = make_classification(
n_samples=1000,
n_features=10,
n_informative=10,
random_state=1111,
n_classes=2,
class_sep=2.5,
n_redundant=0,
)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.1, random_state=1111
)
model = NaiveBayesClassifier()
model.fit(X_train, y_train)
predictions = model.predict(X_test)[:, 1]
print("classification accuracy", roc_auc_score(y_test, predictions))
if __name__ == "__main__":
classification()
examples/nearest_neighbors.py
+59
View File
@@ -0,0 +1,59 @@
try:
from sklearn.model_selection import train_test_split
except ImportError:
from sklearn.cross_validation import train_test_split
from sklearn.datasets import make_classification
from sklearn.datasets import make_regression
from scipy.spatial import distance
from mla import knn
from mla.metrics.metrics import mean_squared_error, accuracy
def regression():
# Generate a random regression problem
X, y = make_regression(
n_samples=500,
n_features=5,
n_informative=5,
n_targets=1,
noise=0.05,
random_state=1111,
bias=0.5,
)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.25, random_state=1111
)
model = knn.KNNRegressor(k=5, distance_func=distance.euclidean)
model.fit(X_train, y_train)
predictions = model.predict(X_test)
print("regression mse", mean_squared_error(y_test, predictions))
def classification():
X, y = make_classification(
n_samples=500,
n_features=5,
n_informative=5,
n_redundant=0,
n_repeated=0,
n_classes=3,
random_state=1111,
class_sep=1.5,
)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.1, random_state=1111
)
clf = knn.KNNClassifier(k=5, distance_func=distance.euclidean)
clf.fit(X_train, y_train)
predictions = clf.predict(X_test)
print("classification accuracy", accuracy(y_test, predictions))
if __name__ == "__main__":
regression()
classification()
examples/nnet_convnet_mnist.py
+57
View File
@@ -0,0 +1,57 @@
import logging
from mla.datasets import load_mnist
from mla.metrics import accuracy
from mla.neuralnet import NeuralNet
from mla.neuralnet.layers import (
Activation,
Convolution,
MaxPooling,
Flatten,
Dropout,
Parameters,
)
from mla.neuralnet.layers import Dense
from mla.neuralnet.optimizers import Adadelta
from mla.utils import one_hot
logging.basicConfig(level=logging.DEBUG)
# Load MNIST dataset
X_train, X_test, y_train, y_test = load_mnist()
# Normalize data
X_train /= 255.0
X_test /= 255.0
y_train = one_hot(y_train.flatten())
y_test = one_hot(y_test.flatten())
print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)
# Approx. 15-20 min. per epoch
model = NeuralNet(
layers=[
Convolution(n_filters=32, filter_shape=(3, 3), padding=(1, 1), stride=(1, 1)),
Activation("relu"),
Convolution(n_filters=32, filter_shape=(3, 3), padding=(1, 1), stride=(1, 1)),
Activation("relu"),
MaxPooling(pool_shape=(2, 2), stride=(2, 2)),
Dropout(0.5),
Flatten(),
Dense(128),
Activation("relu"),
Dropout(0.5),
Dense(10),
Activation("softmax"),
],
loss="categorical_crossentropy",
optimizer=Adadelta(),
metric="accuracy",
batch_size=128,
max_epochs=3,
)
model.fit(X_train, y_train)
predictions = model.predict(X_test)
print(accuracy(y_test, predictions))
examples/nnet_mlp.py
+95
View File
@@ -0,0 +1,95 @@
import logging
try:
from sklearn.model_selection import train_test_split
except ImportError:
from sklearn.cross_validation import train_test_split
from sklearn.datasets import make_classification
from sklearn.datasets import make_regression
from sklearn.metrics import roc_auc_score
from mla.metrics.metrics import mean_squared_error
from mla.neuralnet import NeuralNet
from mla.neuralnet.constraints import MaxNorm
from mla.neuralnet.layers import Activation, Dense, Dropout
from mla.neuralnet.optimizers import Adadelta, Adam
from mla.neuralnet.parameters import Parameters
from mla.neuralnet.regularizers import L2
from mla.utils import one_hot
logging.basicConfig(level=logging.DEBUG)
def classification():
# Generate a random binary classification problem.
X, y = make_classification(
n_samples=1000,
n_features=100,
n_informative=75,
random_state=1111,
n_classes=2,
class_sep=2.5,
)
y = one_hot(y)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.15, random_state=1111
)
model = NeuralNet(
layers=[
Dense(256, Parameters(init="uniform", regularizers={"W": L2(0.05)})),
Activation("relu"),
Dropout(0.5),
Dense(128, Parameters(init="normal", constraints={"W": MaxNorm()})),
Activation("relu"),
Dense(2),
Activation("softmax"),
],
loss="categorical_crossentropy",
optimizer=Adadelta(),
metric="accuracy",
batch_size=64,
max_epochs=25,
)
model.fit(X_train, y_train)
predictions = model.predict(X_test)
print("classification accuracy", roc_auc_score(y_test[:, 0], predictions[:, 0]))
def regression():
# Generate a random regression problem
X, y = make_regression(
n_samples=5000,
n_features=25,
n_informative=25,
n_targets=1,
random_state=100,
noise=0.05,
)
y *= 0.01
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.1, random_state=1111
)
model = NeuralNet(
layers=[
Dense(64, Parameters(init="normal")),
Activation("linear"),
Dense(32, Parameters(init="normal")),
Activation("linear"),
Dense(1),
],
loss="mse",
optimizer=Adam(),
metric="mse",
batch_size=256,
max_epochs=15,
)
model.fit(X_train, y_train)
predictions = model.predict(X_test)
print("regression mse", mean_squared_error(y_test, predictions.flatten()))
if __name__ == "__main__":
classification()
regression()
examples/nnet_rnn_binary_add.py
+78
View File
@@ -0,0 +1,78 @@
import logging
from itertools import combinations, islice
import numpy as np
try:
from sklearn.model_selection import train_test_split
except ImportError:
from sklearn.cross_validation import train_test_split
from mla.metrics import accuracy
from mla.neuralnet import NeuralNet
from mla.neuralnet.layers import Activation, TimeDistributedDense
from mla.neuralnet.layers.recurrent import LSTM
from mla.neuralnet.optimizers import Adam
logging.basicConfig(level=logging.DEBUG)
def addition_dataset(dim=10, n_samples=10000, batch_size=64):
"""Generate binary addition dataset.
http://devankuleindiren.com/Projects/rnn_arithmetic.php
"""
binary_format = "{:0" + str(dim) + "b}"
# Generate all possible number combinations
combs = list(islice(combinations(range(2 ** (dim - 1)), 2), n_samples))
# Initialize empty arrays
X = np.zeros((len(combs), dim, 2), dtype=np.uint8)
y = np.zeros((len(combs), dim, 1), dtype=np.uint8)
for i, (a, b) in enumerate(combs):
# Convert numbers to binary format
X[i, :, 0] = list(reversed([int(x) for x in binary_format.format(a)]))
X[i, :, 1] = list(reversed([int(x) for x in binary_format.format(b)]))
# Generate target variable (a+b)
y[i, :, 0] = list(reversed([int(x) for x in binary_format.format(a + b)]))
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=1111
)
# Round number of examples for batch processing
train_b = (X_train.shape[0] // batch_size) * batch_size
test_b = (X_test.shape[0] // batch_size) * batch_size
X_train = X_train[0:train_b]
y_train = y_train[0:train_b]
X_test = X_test[0:test_b]
y_test = y_test[0:test_b]
return X_train, X_test, y_train, y_test
def addition_problem(ReccurentLayer):
X_train, X_test, y_train, y_test = addition_dataset(8, 5000)
print(X_train.shape, X_test.shape)
model = NeuralNet(
layers=[ReccurentLayer, TimeDistributedDense(1), Activation("sigmoid")],
loss="mse",
optimizer=Adam(),
metric="mse",
batch_size=64,
max_epochs=15,
)
model.fit(X_train, y_train)
predictions = np.round(model.predict(X_test))
predictions = np.packbits(predictions.astype(np.uint8))
y_test = np.packbits(y_test.astype(np.int))
print(accuracy(y_test, predictions))
# RNN
# addition_problem(RNN(16, parameters=Parameters(constraints={'W': SmallNorm(), 'U': SmallNorm()})))
# LSTM
addition_problem(LSTM(16))
examples/nnet_rnn_text_generation.py
+82
View File
@@ -0,0 +1,82 @@
from __future__ import print_function
import logging
import random
import numpy as np
import sys
from mla.datasets import load_nietzsche
from mla.neuralnet import NeuralNet
from mla.neuralnet.constraints import SmallNorm
from mla.neuralnet.layers import Activation, Dense
from mla.neuralnet.layers.recurrent import LSTM, RNN
from mla.neuralnet.optimizers import RMSprop
logging.basicConfig(level=logging.DEBUG)
# Example taken from: https://github.com/fchollet/keras/blob/master/examples/lstm_text_generation.py
def sample(preds, temperature=1.0):
# helper function to sample an index from a probability array
preds = np.asarray(preds).astype("float64")
preds = np.log(preds) / temperature
exp_preds = np.exp(preds)
preds = exp_preds / np.sum(exp_preds)
probas = np.random.multinomial(1, preds, 1)
return np.argmax(probas)
X, y, text, chars, char_indices, indices_char = load_nietzsche()
# Round the number of sequences for batch processing
items_count = X.shape[0] - (X.shape[0] % 64)
maxlen = X.shape[1]
X = X[0:items_count]
y = y[0:items_count]
print(X.shape, y.shape)
# LSTM OR RNN
# rnn_layer = RNN(128, return_sequences=False)
rnn_layer = LSTM(128, return_sequences=False)
model = NeuralNet(
layers=[
rnn_layer,
# Flatten(),
# TimeStepSlicer(-1),
Dense(X.shape[2]),
Activation("softmax"),
],
loss="categorical_crossentropy",
optimizer=RMSprop(learning_rate=0.01),
metric="accuracy",
batch_size=64,
max_epochs=1,
shuffle=False,
)
for _ in range(25):
model.fit(X, y)
start_index = random.randint(0, len(text) - maxlen - 1)
generated = ""
sentence = text[start_index : start_index + maxlen]
generated += sentence
print('----- Generating with seed: "' + sentence + '"')
sys.stdout.write(generated)
for i in range(100):
x = np.zeros((64, maxlen, len(chars)))
for t, char in enumerate(sentence):
x[0, t, char_indices[char]] = 1.0
preds = model.predict(x)[0]
next_index = sample(preds, 0.5)
next_char = indices_char[next_index]
generated += next_char
sentence = sentence[1:] + next_char
sys.stdout.write(next_char)
sys.stdout.flush()
print()
examples/pca.py
+39
View File
@@ -0,0 +1,39 @@
try:
from sklearn.model_selection import train_test_split
except ImportError:
from sklearn.cross_validation import train_test_split
from sklearn.datasets import make_classification
from mla.linear_models import LogisticRegression
from mla.metrics import accuracy
from mla.pca import PCA
# logging.basicConfig(level=logging.DEBUG)
# Generate a random binary classification problem.
X, y = make_classification(
n_samples=1000,
n_features=100,
n_informative=75,
random_state=1111,
n_classes=2,
class_sep=2.5,
)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.25, random_state=1111
)
for s in ["svd", "eigen"]:
p = PCA(15, solver=s)
# fit PCA with training data, not entire dataset
p.fit(X_train)
X_train_reduced = p.transform(X_train)
X_test_reduced = p.transform(X_test)
model = LogisticRegression(lr=0.001, max_iters=2500)
model.fit(X_train_reduced, y_train)
predictions = model.predict(X_test_reduced)
print("Classification accuracy for %s PCA: %s" % (s, accuracy(y_test, predictions)))
examples/random_forest.py
+71
View File
@@ -0,0 +1,71 @@
import logging
import numpy as np
from sklearn.datasets import make_classification
from sklearn.datasets import make_regression
from sklearn.metrics import roc_auc_score, accuracy_score
try:
from sklearn.model_selection import train_test_split
except ImportError:
from sklearn.cross_validation import train_test_split
from mla.ensemble.random_forest import RandomForestClassifier, RandomForestRegressor
from mla.metrics.metrics import mean_squared_error
logging.basicConfig(level=logging.DEBUG)
def classification():
# Generate a random binary classification problem.
X, y = make_classification(
n_samples=500,
n_features=10,
n_informative=10,
random_state=1111,
n_classes=2,
class_sep=2.5,
n_redundant=0,
)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.15, random_state=1111
)
model = RandomForestClassifier(n_estimators=10, max_depth=4)
model.fit(X_train, y_train)
predictions_prob = model.predict(X_test)[:, 1]
predictions = np.argmax(model.predict(X_test), axis=1)
# print(predictions.shape)
print("classification, roc auc score: %s" % roc_auc_score(y_test, predictions_prob))
print("classification, accuracy score: %s" % accuracy_score(y_test, predictions))
def regression():
# Generate a random regression problem
X, y = make_regression(
n_samples=500,
n_features=5,
n_informative=5,
n_targets=1,
noise=0.05,
random_state=1111,
bias=0.5,
)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.1, random_state=1111
)
model = RandomForestRegressor(n_estimators=50, max_depth=10, max_features=3)
model.fit(X_train, y_train)
predictions = model.predict(X_test)
print(
"regression, mse: %s"
% mean_squared_error(y_test.flatten(), predictions.flatten())
)
if __name__ == "__main__":
classification()
# regression()
examples/rbm.py
+25
View File
@@ -0,0 +1,25 @@
import logging
import numpy as np
from mla.rbm import RBM
logging.basicConfig(level=logging.DEBUG)
def print_curve(rbm):
from matplotlib import pyplot as plt
def moving_average(a, n=25):
ret = np.cumsum(a, dtype=float)
ret[n:] = ret[n:] - ret[:-n]
return ret[n - 1 :] / n
plt.plot(moving_average(rbm.errors))
plt.show()
X = np.random.uniform(0, 1, (1500, 10))
rbm = RBM(n_hidden=10, max_epochs=200, batch_size=10, learning_rate=0.1)
rbm.fit(X)
print_curve(rbm)
examples/rl_deep_q_learning.py
+37
View File
@@ -0,0 +1,37 @@
import logging
from mla.neuralnet import NeuralNet
from mla.neuralnet.layers import Activation, Dense
from mla.neuralnet.optimizers import Adam
from mla.rl.dqn import DQN
logging.basicConfig(level=logging.CRITICAL)
def mlp_model(n_actions, batch_size=64):
model = NeuralNet(
layers=[Dense(32), Activation("relu"), Dense(n_actions)],
loss="mse",
optimizer=Adam(),
metric="mse",
batch_size=batch_size,
max_epochs=1,
verbose=False,
)
return model
model = DQN(n_episodes=2500, batch_size=64)
model.init_environment("CartPole-v0")
model.init_model(mlp_model)
try:
# Train the model
# It can take from 300 to 2500 episodes to solve CartPole-v0 problem due to randomness of environment.
# You can stop training process using Ctrl+C signal
# Read more about this problem: https://gym.openai.com/envs/CartPole-v0
model.train(render=False)
except KeyboardInterrupt:
pass
# Render trained model
model.play(episodes=100)
examples/svm.py
+42
View File
@@ -0,0 +1,42 @@
import logging
try:
from sklearn.model_selection import train_test_split
except ImportError:
from sklearn.cross_validation import train_test_split
from sklearn.datasets import make_classification
from mla.metrics.metrics import accuracy
from mla.svm.kernerls import Linear, RBF
from mla.svm.svm import SVM
logging.basicConfig(level=logging.DEBUG)
def classification():
# Generate a random binary classification problem.
X, y = make_classification(
n_samples=1200,
n_features=10,
n_informative=5,
random_state=1111,
n_classes=2,
class_sep=1.75,
)
# Convert y to {-1, 1}
y = (y * 2) - 1
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=1111
)
for kernel in [RBF(gamma=0.1), Linear()]:
model = SVM(max_iter=500, kernel=kernel, C=0.6)
model.fit(X_train, y_train)
predictions = model.predict(X_test)
print(
"Classification accuracy (%s): %s" % (kernel, accuracy(y_test, predictions))
)
if __name__ == "__main__":
classification()
examples/t-sne.py
+28
View File
@@ -0,0 +1,28 @@
import logging
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from mla.tsne import TSNE
logging.basicConfig(level=logging.DEBUG)
X, y = make_classification(
n_samples=500,
n_features=10,
n_informative=5,
n_redundant=0,
random_state=1111,
n_classes=2,
class_sep=2.5,
)
p = TSNE(2, max_iter=500)
X = p.fit_transform(X)
colors = ["red", "green"]
for t in range(2):
t_mask = (y == t).astype(bool)
plt.scatter(X[t_mask, 0], X[t_mask, 1], color=colors[t])
plt.show()