Naïve Bayes Classifier

The Story That Explains Naïve Bayes

The Foundation — Bayes' Theorem

Everything in Naïve Bayes flows from one equation discovered by the Reverend Thomas Bayes in the 18th century. It tells you how to update a belief when you receive new evidence.

The Naïve Independence Assumption

With many features, computing P(word₁, word₂, word₃, ... | Class) directly is impossible — you would need to have seen every exact combination of words in training. Naïve Bayes solves this with a bold simplification.

Step-by-Step Manual Calculation

Let us classify one new email by hand using a tiny training set of 10 emails with three binary features.

New email to classify: contains "free" ✅, "winner" ✅, "meeting" ❌. Spam or Ham?

The Three Variants of Naïve Bayes

Naïve Bayes is not a single algorithm — it is a family. The variant you choose depends entirely on the type of features in your data.

The Zero Frequency Problem — Laplace Smoothing

Gaussian NB — How Continuous Features Are Modelled

When features are continuous (like age, glucose level, temperature), counting occurrences makes no sense. Gaussian NB instead models each feature as a normal distribution per class — estimating μ and σ² from training data.

Python Implementation

Gaussian Naïve Bayes — Iris Classification

from sklearn.naive_bayes import GaussianNB
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.datasets import load_iris
import numpy as np

# Load Iris — continuous features → Gaussian NB is ideal
iris = load_iris()
X, y = iris.data, iris.target

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

gnb = GaussianNB(
    var_smoothing=1e-9   # small constant added to variance for stability
)
gnb.fit(X_train, y_train)

y_pred = gnb.predict(X_test)
print(f"Accuracy: {gnb.score(X_test, y_test):.4f}")
print(classification_report(y_test, y_pred, target_names=iris.target_names))

# Inspect learned μ and σ² per class per feature
for i, cls in enumerate(iris.target_names):
    print(f"\n{cls}:")
    for j, feat in enumerate(iris.feature_names):
        print(f"  {feat:30s}  μ={gnb.theta_[i,j]:.3f}  σ²={gnb.var_[i,j]:.4f}")

Multinomial NB — Spam Detection Pipeline

from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.metrics import classification_report

emails = [
    "Get free money now win prizes",
    "Congratulations you are a winner claim your reward",
    "Free entry in our lucky draw text WIN to 88888",
    "Hi how are you doing today everything okay",
    "Meeting scheduled for 3pm please confirm attendance",
    "Project update attached please review the report",
    "WINNER you have been selected for a cash prize",
    "Lunch tomorrow at the usual place let me know",
    "Urgent your account will be suspended click here now",
    "Can we reschedule the team standup to Friday afternoon",
]
labels = [1, 1, 1, 0, 0, 0, 1, 0, 1, 0]  # 1=spam 0=ham

X_train, X_test, y_train, y_test = train_test_split(
    emails, labels, test_size=0.3, random_state=42
)

# TF-IDF → Multinomial NB pipeline
pipeline = Pipeline([
    ('tfidf', TfidfVectorizer(
        stop_words='english',
        ngram_range=(1, 2),    # unigrams + bigrams
        min_df=1
    )),
    ('clf', MultinomialNB(alpha=1.0))   # Laplace smoothing
])

pipeline.fit(X_train, y_train)

# Predict with class probabilities
y_pred = pipeline.predict(X_test)
y_prob = pipeline.predict_proba(X_test)

for email, pred, prob in zip(X_test, y_pred, y_prob):
    label = "SPAM" if pred == 1 else "HAM "
    print(f"[{label}] ({prob[1]*100:5.1f}% spam)  {email[:50]}")

Bernoulli NB — Binary Bag-of-Words

from sklearn.naive_bayes import BernoulliNB
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.pipeline import Pipeline

# binary=True: CountVectorizer caps all counts at 1 (word present/absent)
pipeline_bnb = Pipeline([
    ('bow', CountVectorizer(binary=True, stop_words='english')),
    ('clf', BernoulliNB(alpha=1.0, binarize=None))
    # binarize=None: features already binarised by CountVectorizer
])

pipeline_bnb.fit(X_train, y_train)
print("BernoulliNB accuracy:", f"{pipeline_bnb.score(X_test, y_test):.4f}")

Comparing All Variants with Cross-Validation

from sklearn.naive_bayes import ComplementNB
from sklearn.model_selection import cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer

# ComplementNB — best for imbalanced class distributions
pipeline_cnb = Pipeline([
    ('tfidf', TfidfVectorizer(stop_words='english')),
    ('clf',   ComplementNB(alpha=1.0, norm=True))
])

# 3-fold CV on the full emails dataset
for name, pipe in [
    ('MultinomialNB', pipeline),
    ('BernoulliNB  ', pipeline_bnb),
    ('ComplementNB ', pipeline_cnb)
]:
    scores = cross_val_score(pipe, emails, labels, cv=3, scoring='f1')
    print(f"{name}: F1 = {scores.mean():.3f} ± {scores.std():.3f}")

Full Real-World NLP Pipeline — 20 Newsgroups

from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.metrics import classification_report, ConfusionMatrixDisplay
from sklearn.datasets import fetch_20newsgroups
import matplotlib.pyplot as plt

# 4-topic multi-class classification
categories = ['sci.space', 'comp.graphics',
              'rec.sport.hockey', 'talk.politics.guns']

train = fetch_20newsgroups(subset='train', categories=categories)
test  = fetch_20newsgroups(subset='test',  categories=categories)

# Pipeline with hyperparameter search
pipeline = Pipeline([
    ('tfidf', TfidfVectorizer()),
    ('clf',   MultinomialNB())
])

param_grid = {
    'tfidf__max_features': [5000, 20000, None],
    'tfidf__ngram_range':  [(1, 1), (1, 2)],
    'tfidf__sublinear_tf': [True, False],
    'clf__alpha':          [0.01, 0.1, 0.5, 1.0],
}

grid = GridSearchCV(
    pipeline, param_grid,
    cv=5, scoring='f1_macro',
    n_jobs=-1, verbose=0
)
grid.fit(train.data, train.target)

print("Best params:", grid.best_params_)
print(f"Best CV F1:   {grid.best_score_:.4f}")

y_pred = grid.predict(test.data)
print(classification_report(
    test.target, y_pred,
    target_names=train.target_names
))

# Confusion matrix
ConfusionMatrixDisplay.from_predictions(
    test.target, y_pred,
    display_labels=train.target_names,
    xticks_rotation='vertical'
)
plt.tight_layout()
plt.show()

When to Use Naïve Bayes

Naïve Bayes vs Other Classifiers

Strengths vs Weaknesses

Naïve Bayes in the Real World

Golden Rules