Bag of Words: CountVectorizer and TfidfVectorizer

The Text-to-Numbers Problem

# Bag of Words: ignore order, just count words
doc1 = 'the cat sat on the mat'
doc2 = 'the cat ate the rat'

# Vocabulary: all unique words across documents
vocab = sorted(set(doc1.split() + doc2.split()))
print('Vocabulary:', vocab)

# Count vectors
vec1 = [doc1.split().count(w) for w in vocab]
vec2 = [doc2.split().count(w) for w in vocab]
print('doc1 vector:', vec1)
print('doc2 vector:', vec2)

CountVectorizer: Building the Vocabulary

from sklearn.feature_extraction.text import CountVectorizer

corpus = [
    'I love Python and machine learning',
    'Machine learning is awesome',
    'Python is great for data science',
    'I hate bugs in Python code'
]

vec = CountVectorizer()
X = vec.fit_transform(corpus)  # Returns sparse matrix

print('Vocabulary size:', len(vec.vocabulary_))
print('Matrix shape:', X.shape)  # (4 docs, N vocabulary words)
print('Vocabulary:', sorted(vec.vocabulary_.keys()))

Sparse Matrices and Memory Efficiency

from sklearn.feature_extraction.text import CountVectorizer
from scipy.sparse import issparse
import numpy as np

corpus = ['Python is great', 'Machine learning rocks', 'Data science rules']
vec = CountVectorizer()
X_sparse = vec.fit_transform(corpus)

print('Is sparse:', issparse(X_sparse))        # True
print('Shape:', X_sparse.shape)
print('Non-zero entries:', X_sparse.nnz)       # Only non-zero values stored
print('Density:', X_sparse.nnz / np.prod(X_sparse.shape))  # Very low

# Dense (RAM-expensive for large corpora)
X_dense = X_sparse.toarray()
print('Dense shape:', X_dense.shape)

Preprocessing: Stopwords, Lowercase, and n-grams

from sklearn.feature_extraction.text import CountVectorizer

corpus = ['not good at all', 'very good movie', 'bad experience']

# Unigrams only (default)
uni_vec = CountVectorizer(stop_words='english')
print('Unigram features:', uni_vec.fit(corpus).get_feature_names_out())

# Bigrams too
bigram_vec = CountVectorizer(ngram_range=(1,2), stop_words='english')
bigram_features = bigram_vec.fit(corpus).get_feature_names_out()
print('Unigram+Bigram features:', bigram_features)
# 'not good' appears as a bigram -- captures negation

The Problem with Raw Counts: Common Words

from sklearn.feature_extraction.text import CountVectorizer
import numpy as np

corpus = [
    'the cat sat on the mat the cat is fat',
    'the eigenvalue decomposition is powerful math'
]

vec = CountVectorizer()
X = vec.fit_transform(corpus).toarray()
features = vec.get_feature_names_out()

for i, doc in enumerate(corpus[:1]):
    counts = sorted(zip(features, X[i]), key=lambda x: -x[1])
    print('Top words by count in doc 1:')
    for word, count in counts[:5]:
        print(f'  {word}: {count}')
# 'the' dominates even though it is meaningless

TF-IDF: Term Frequency Inverse Document Frequency

import numpy as np

# Manual TF-IDF
documents = [
    'cat sat mat cat',   # 'cat' appears twice here
    'dog ran park',
    'cat dog park'
]

N = len(documents)  # 3 documents

# IDF for 'cat': appears in documents 0 and 2 (df=2)
df_cat = 2
idf_cat = np.log((N + 1) / (df_cat + 1)) + 1

# TF for 'cat' in doc0: 2 out of 4 words
tf_cat_doc0 = 2 / 4

tfidf_cat_doc0 = tf_cat_doc0 * idf_cat
print(f'TF-IDF(cat, doc0) = {tfidf_cat_doc0:.4f}')

TfidfVectorizer: One-Step Text Transformation

from sklearn.feature_extraction.text import TfidfVectorizer

corpus = [
    'I love Python and machine learning',
    'Machine learning is awesome',
    'Python is great for data science',
    'I hate bugs in Python code'
]

tfidf = TfidfVectorizer(stop_words='english', max_features=20)
X = tfidf.fit_transform(corpus)

print('Shape:', X.shape)
print('Features:', tfidf.get_feature_names_out())

# Inspect TF-IDF weights for document 0
import numpy as np
weights = zip(tfidf.get_feature_names_out(), X.toarray()[0])
for word, weight in sorted(weights, key=lambda x: -x[1])[:5]:
    print(f'  {word}: {weight:.4f}')

max_features and vocabulary Parameters

from sklearn.feature_extraction.text import TfidfVectorizer

corpus = ['...']  # assume large corpus

tfidf = TfidfVectorizer(
    max_features=50000,   # Top 50k words by frequency
    min_df=5,            # Ignore words in fewer than 5 documents
    max_df=0.95,         # Ignore words in more than 95% of docs
    ngram_range=(1, 2),  # Include bigrams
    stop_words='english',
    sublinear_tf=True    # Replace TF with 1+log(TF) to dampen outliers
)

print('CountVectorizer vs TfidfVectorizer settings configured')
print('sublinear_tf=True: dampens high-frequency words further')

CountVectorizer vs TfidfVectorizer: When to Use Each

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score

corpus = ['spam message buy now', 'hello friend how are you',
          'click here to buy', 'good morning everyone']
labels = [1, 0, 1, 0]  # 1=spam, 0=ham

# Naive Bayes: works well with CountVectorizer
nb_pipe = Pipeline([('vec', CountVectorizer()), ('clf', MultinomialNB())])

# Logistic regression: works better with TF-IDF
lr_pipe = Pipeline([('vec', TfidfVectorizer()), ('clf', LogisticRegression())])

print('CountVectorizer + MultinomialNB: standard for text Naive Bayes')
print('TfidfVectorizer + LogisticRegression: standard for text linear models')

Inspecting the Document-Term Matrix

from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd

corpus = [
    'Python machine learning tutorial',
    'Deep learning neural networks',
    'Python data analysis pandas numpy'
]

tfidf = TfidfVectorizer()
X = tfidf.fit_transform(corpus)

# Convert to DataFrame for inspection
df = pd.DataFrame(
    X.toarray(),
    columns=tfidf.get_feature_names_out()
)

print('Document-Term Matrix (TF-IDF weights):')
print(df.to_string())

Character-Level N-grams for Robust Tokenisation

from sklearn.feature_extraction.text import TfidfVectorizer

corpus = [
    'python programming language',
    'pythn programmng (typo)',  # Misspellings
    'java programming'
]

# Word-level (default): typos create unseen tokens
word_vec = TfidfVectorizer(analyzer='word')
word_features = word_vec.fit(corpus).get_feature_names_out()
print('Word features:', word_features)

# Character n-gram: handles typos gracefully
char_vec = TfidfVectorizer(analyzer='char_wb', ngram_range=(3, 4))
X_char = char_vec.fit_transform(corpus)
print('Char n-gram features:', len(char_vec.get_feature_names_out()), 'total')
print('Typos share substrings with correct words -> similar vectors')

Quick Check

Lesson Recap