Simplify cleaning transformers and shorten module names

Move cleaning transformers into classifiers/cleaning.py (dropping the
separate cleaning package) and implement them as plain classes rather
than BaseEstimator/TransformerMixin subclasses, since Pipeline only
needs fit/transform via duck typing. Also rename feature_classifier.py
and bag_of_words.py to features.py and bow.py for brevity.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Chris Proctor
2026-06-08 10:02:41 -04:00
parent 5f6f171369
commit bbe8054910
6 changed files with 6 additions and 10 deletions

41
classifiers/cleaning.py Normal file
View File

@@ -0,0 +1,41 @@
import re
import numpy as np
STOPWORDS = {
"a", "an", "the", "is", "it", "in", "on", "at", "to", "for",
"of", "and", "or", "but", "not", "with", "as", "by", "from",
"this", "that", "was", "are", "be", "been", "have", "has",
"had", "do", "did", "will", "would", "could", "should",
"i", "me", "my", "you", "your", "he", "she", "we", "they",
"his", "her", "our", "their", "its", "what", "which",
}
class LowercaseTransformer:
def fit(self, X, y=None):
return self
def transform(self, X):
return np.array([msg.lower() for msg in X])
class StopwordRemover:
def fit(self, X, y=None):
return self
def transform(self, X):
return np.array([self._remove(msg) for msg in X])
def _remove(self, message):
words = message.split()
return " ".join(w for w in words if w.lower() not in STOPWORDS)
class PunctuationRemover:
def fit(self, X, y=None):
return self
def transform(self, X):
return np.array([re.sub(r"[^\w\s]", " ", msg) for msg in X])