wehub-resource-sync
291 lines
8.3 KiB
Python
291 lines
8.3 KiB
Python
# Sebastian Raschka, 2015 (http://sebastianraschka.com)
|
|
# Python Machine Learning - Code Examples
|
|
#
|
|
# Chapter 8 - Applying Machine Learning To Sentiment Analysis
|
|
#
|
|
# S. Raschka. Python Machine Learning. Packt Publishing Ltd., 2015.
|
|
# GitHub Repo: https://github.com/rasbt/python-machine-learning-book
|
|
#
|
|
# License: MIT
|
|
# https://github.com/rasbt/python-machine-learning-book/blob/master/LICENSE.txt
|
|
|
|
import pyprind
|
|
import pandas as pd
|
|
import os
|
|
import numpy as np
|
|
import re
|
|
import nltk
|
|
from sklearn.feature_extraction.text import CountVectorizer
|
|
from sklearn.feature_extraction.text import TfidfTransformer
|
|
from sklearn.pipeline import Pipeline
|
|
from sklearn.linear_model import LogisticRegression
|
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
from sklearn.feature_extraction.text import HashingVectorizer
|
|
from sklearn.linear_model import SGDClassifier
|
|
from nltk.stem.porter import PorterStemmer
|
|
from nltk.corpus import stopwords
|
|
|
|
# Added version check for recent scikit-learn 0.18 checks
|
|
from distutils.version import LooseVersion as Version
|
|
from sklearn import __version__ as sklearn_version
|
|
if Version(sklearn_version) < '0.18':
|
|
from sklearn.cross_validation import GridSearchCV
|
|
else:
|
|
from sklearn.model_selection import GridSearchCV
|
|
|
|
#############################################################################
|
|
print(50 * '=')
|
|
print('Section: Obtaining the IMDb movie review dataset')
|
|
print(50 * '-')
|
|
|
|
print('!! This script assumes that the movie dataset is located in the'
|
|
' current directory under ./aclImdb')
|
|
|
|
_ = input('Please hit enter to continue.')
|
|
|
|
basepath = './aclImdb'
|
|
|
|
"""
|
|
labels = {'pos': 1, 'neg': 0}
|
|
pbar = pyprind.ProgBar(50000)
|
|
df = pd.DataFrame()
|
|
for s in ('test', 'train'):
|
|
for l in ('pos', 'neg'):
|
|
path = os.path.join(basepath, s, l)
|
|
for file in os.listdir(path):
|
|
with open(os.path.join(path, file), 'r',
|
|
encoding='utf-8') as infile:
|
|
txt = infile.read()
|
|
df = df.append([[txt, labels[l]]], ignore_index=True)
|
|
pbar.update()
|
|
df.columns = ['review', 'sentiment']
|
|
|
|
|
|
np.random.seed(0)
|
|
df = df.reindex(np.random.permutation(df.index))
|
|
|
|
df.to_csv('./movie_data.csv', index=False)
|
|
"""
|
|
|
|
df = pd.read_csv('../datasets/movie/movie_data.csv')
|
|
print('Excerpt of the movie dataset', df.head(3))
|
|
|
|
|
|
#############################################################################
|
|
print(50 * '=')
|
|
print('Section: Transforming documents into feature vectors')
|
|
print(50 * '-')
|
|
|
|
count = CountVectorizer()
|
|
docs = np.array(['The sun is shining',
|
|
'The weather is sweet',
|
|
'The sun is shining and the weather is sweet'])
|
|
bag = count.fit_transform(docs)
|
|
|
|
print('Vocabulary', count.vocabulary_)
|
|
print('bag.toarray()', bag.toarray())
|
|
|
|
|
|
#############################################################################
|
|
print(50 * '=')
|
|
print('Section: Assessing word relevancy via term frequency-inverse'
|
|
' document frequency')
|
|
print(50 * '-')
|
|
|
|
np.set_printoptions(precision=2)
|
|
tfidf = TfidfTransformer(use_idf=True, norm='l2', smooth_idf=True)
|
|
print(tfidf.fit_transform(count.fit_transform(docs)).toarray())
|
|
|
|
tf_is = 2
|
|
n_docs = 3
|
|
idf_is = np.log((n_docs + 1) / (3 + 1))
|
|
tfidf_is = tf_is * (idf_is + 1)
|
|
print('tf-idf of term "is" = %.2f' % tfidf_is)
|
|
|
|
|
|
tfidf = TfidfTransformer(use_idf=True, norm=None, smooth_idf=True)
|
|
raw_tfidf = tfidf.fit_transform(count.fit_transform(docs)).toarray()[-1]
|
|
print('raw tf-idf', raw_tfidf)
|
|
|
|
l2_tfidf = raw_tfidf / np.sqrt(np.sum(raw_tfidf**2))
|
|
l2_tfidf
|
|
print('l2 tf-idf', l2_tfidf)
|
|
|
|
|
|
#############################################################################
|
|
print(50 * '=')
|
|
print('Section: Cleaning text data')
|
|
print(50 * '-')
|
|
|
|
print('Excerpt:\n\n', df.loc[0, 'review'][-50:])
|
|
|
|
|
|
def preprocessor(text):
|
|
text = re.sub('<[^>]*>', '', text)
|
|
emoticons = re.findall('(?::|;|=)(?:-)?(?:\)|\(|D|P)', text)
|
|
text = re.sub('[\W]+', ' ', text.lower()) +\
|
|
' '.join(emoticons).replace('-', '')
|
|
return text
|
|
|
|
|
|
print('Preprocessor on Excerpt:\n\n', preprocessor(df.loc[0, 'review'][-50:]))
|
|
|
|
res = preprocessor("</a>This :) is :( a test :-)!")
|
|
print('Preprocessor on "</a>This :) is :( a test :-)!":\n\n', res)
|
|
|
|
df['review'] = df['review'].apply(preprocessor)
|
|
|
|
|
|
#############################################################################
|
|
print(50 * '=')
|
|
print('Section: Processing documents into tokens')
|
|
print(50 * '-')
|
|
|
|
porter = PorterStemmer()
|
|
|
|
|
|
def tokenizer(text):
|
|
return text.split()
|
|
|
|
|
|
def tokenizer_porter(text):
|
|
return [porter.stem(word) for word in text.split()]
|
|
|
|
|
|
t1 = tokenizer('runners like running and thus they run')
|
|
print("Tokenize: 'runners like running and thus they run'")
|
|
print(t1)
|
|
|
|
t2 = tokenizer_porter('runners like running and thus they run')
|
|
print("\nPorter-Tokenize: 'runners like running and thus they run'")
|
|
print(t2)
|
|
|
|
nltk.download('stopwords')
|
|
|
|
|
|
print('remove stop words')
|
|
stop = stopwords.words('english')
|
|
|
|
r = [w for w in tokenizer_porter('a runner likes running and runs a lot')[-10:]
|
|
if w not in stop]
|
|
|
|
print(r)
|
|
|
|
|
|
#############################################################################
|
|
print(50 * '=')
|
|
print('Section: Training a logistic regression model'
|
|
' for document classification')
|
|
print(50 * '-')
|
|
|
|
|
|
X_train = df.loc[:25000, 'review'].values
|
|
y_train = df.loc[:25000, 'sentiment'].values
|
|
X_test = df.loc[25000:, 'review'].values
|
|
y_test = df.loc[25000:, 'sentiment'].values
|
|
|
|
|
|
tfidf = TfidfVectorizer(strip_accents=None,
|
|
lowercase=False,
|
|
preprocessor=None)
|
|
|
|
param_grid = [{'vect__ngram_range': [(1, 1)],
|
|
'vect__stop_words': [stop, None],
|
|
'vect__tokenizer': [tokenizer, tokenizer_porter],
|
|
'clf__penalty': ['l1', 'l2'],
|
|
'clf__C': [1.0, 10.0, 100.0]},
|
|
{'vect__ngram_range': [(1, 1)],
|
|
'vect__stop_words': [stop, None],
|
|
'vect__tokenizer': [tokenizer, tokenizer_porter],
|
|
'vect__use_idf':[False],
|
|
'vect__norm':[None],
|
|
'clf__penalty': ['l1', 'l2'],
|
|
'clf__C': [1.0, 10.0, 100.0]},
|
|
]
|
|
|
|
lr_tfidf = Pipeline([('vect', tfidf),
|
|
('clf', LogisticRegression(random_state=0))])
|
|
|
|
gs_lr_tfidf = GridSearchCV(lr_tfidf, param_grid,
|
|
scoring='accuracy',
|
|
cv=5,
|
|
verbose=1,
|
|
n_jobs=-1)
|
|
|
|
gs_lr_tfidf.fit(X_train, y_train)
|
|
|
|
print('Best parameter set: %s ' % gs_lr_tfidf.best_params_)
|
|
print('CV Accuracy: %.3f' % gs_lr_tfidf.best_score_)
|
|
|
|
|
|
clf = gs_lr_tfidf.best_estimator_
|
|
print('Test Accuracy: %.3f' % clf.score(X_test, y_test))
|
|
|
|
|
|
#############################################################################
|
|
print(50 * '=')
|
|
print('Section: Working with bigger data - online'
|
|
' algorithms and out-of-core learning')
|
|
print(50 * '-')
|
|
|
|
stop = stopwords.words('english')
|
|
|
|
|
|
def tokenizer(text):
|
|
text = re.sub('<[^>]*>', '', text)
|
|
emoticons = re.findall('(?::|;|=)(?:-)?(?:\)|\(|D|P)', text.lower())
|
|
text = re.sub('[\W]+', ' ', text.lower()) +\
|
|
' '.join(emoticons).replace('-', '')
|
|
tokenized = [w for w in text.split() if w not in stop]
|
|
return tokenized
|
|
|
|
|
|
def stream_docs(path):
|
|
with open(path, 'r', encoding='utf-8') as csv:
|
|
next(csv) # skip header
|
|
for line in csv:
|
|
text, label = line[:-3], int(line[-2])
|
|
yield text, label
|
|
|
|
|
|
next(stream_docs(path='./movie_data.csv'))
|
|
|
|
|
|
def get_minibatch(doc_stream, size):
|
|
docs, y = [], []
|
|
try:
|
|
for _ in range(size):
|
|
text, label = next(doc_stream)
|
|
docs.append(text)
|
|
y.append(label)
|
|
except StopIteration:
|
|
return None, None
|
|
return docs, y
|
|
|
|
|
|
vect = HashingVectorizer(decode_error='ignore',
|
|
n_features=2**21,
|
|
preprocessor=None,
|
|
tokenizer=tokenizer)
|
|
|
|
clf = SGDClassifier(loss='log', random_state=1, n_iter=1)
|
|
doc_stream = stream_docs(path='./movie_data.csv')
|
|
|
|
pbar = pyprind.ProgBar(45)
|
|
|
|
classes = np.array([0, 1])
|
|
for _ in range(45):
|
|
X_train, y_train = get_minibatch(doc_stream, size=1000)
|
|
if not X_train:
|
|
break
|
|
X_train = vect.transform(X_train)
|
|
clf.partial_fit(X_train, y_train, classes=classes)
|
|
pbar.update()
|
|
|
|
|
|
X_test, y_test = get_minibatch(doc_stream, size=5000)
|
|
X_test = vect.transform(X_test)
|
|
print('Accuracy: %.3f' % clf.score(X_test, y_test))
|
|
|
|
clf = clf.partial_fit(X_test, y_test)
|