chore: import upstream snapshot with attribution

2026-07-13 13:38:23 +08:00
commit 2725b63d23
700 changed files with 53581 additions and 0 deletions
@@ -0,0 +1,22 @@
+from sklearn.feature_extraction.text import HashingVectorizer
+import re
+import os
+import pickle
+
+
+stop = pickle.load(open('stopwords.pkl', 'rb'))
+
+
+def tokenizer(text):
+    text = re.sub('<[^>]*>', '', text)
+    emoticons = re.findall('(?::|;|=)(?:-)?(?:\)|\(|D|P)',
+                           text.lower())
+    text = re.sub('[\W]+', ' ', text.lower()) + \
+        ' '.join(emoticons).replace('-', '')
+    tokenized = [w for w in text.split() if w not in stop]
+    return tokenized
+
+vect = HashingVectorizer(decode_error='ignore',
+                         n_features=2**21,
+                         preprocessor=None,
+                         tokenizer=tokenizer)