Support Vector Machine

The Story That Explains SVM

The Hyperplane — What SVM Is Drawing

In 2D, a decision boundary is a line. In 3D it is a plane. In higher dimensions it is a hyperplane — the same concept, just harder to visualise. SVM works identically regardless of dimensionality.

Support Vectors — The Only Points That Matter

The SVM Objective — What Is Being Optimised

SVM is fundamentally an optimisation problem. It finds the weight vector w and bias b that define the widest possible margin while correctly classifying all training points.

Hard Margin vs Soft Margin — The C Parameter

The soft margin SVM introduces slack variables ξᵢ that allow some points to violate the margin — at a cost controlled by the hyperparameter C.

The Kernel Trick — Handling Non-Linear Data

The Four Kernels — Choosing the Right One

Python Implementation

Basic SVC — Iris Dataset with All Kernels

from sklearn.svm import SVC
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import classification_report
from sklearn.pipeline import Pipeline

iris = load_iris()
X, y = iris.data, iris.target

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

# ⚠️ CRITICAL: Always scale features before SVM
# SVM is distance-based — unscaled features dominate unfairly
kernels = ['linear', 'rbf', 'poly', 'sigmoid']

for kernel in kernels:
    pipe = Pipeline([
        ('scaler', StandardScaler()),
        ('svc', SVC(
            kernel=kernel,
            C=1.0,
            gamma='scale',   # 1/(n_features × var(X))
            random_state=42
        ))
    ])
    pipe.fit(X_train, y_train)
    acc = pipe.score(X_test, y_test)
    nsv = pipe['svc'].n_support_.sum()  # total support vectors
    print(f"Kernel: {kernel:8s} | Accuracy: {acc:.4f} | Support Vectors: {nsv}")

Inspecting the Fitted Model

# Fit a single model and inspect internals
pipe = Pipeline([
    ('scaler', StandardScaler()),
    ('svc',    SVC(kernel='rbf', C=1.0, gamma='scale',
                    probability=True, random_state=42))
])
pipe.fit(X_train, y_train)

svc = pipe['svc']

print("Support vectors per class:", svc.n_support_)
print("Total support vectors:    ", svc.n_support_.sum())
print("Support vector shape:     ", svc.support_vectors_.shape)

# Predict probabilities (requires probability=True at fit time)
y_prob = pipe.predict_proba(X_test)
y_pred = pipe.predict(X_test)

print("\nSample predictions:")
for i in range(4):
    cls = iris.target_names[y_pred[i]]
    conf = y_prob[i].max() * 100
    print(f"  Predicted: {cls:12s} ({conf:.1f}% confidence)")

Hyperparameter Tuning — GridSearchCV

from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
import numpy as np

pipe = Pipeline([
    ('scaler', StandardScaler()),
    ('svc',    SVC(random_state=42))
])

# Search over C and gamma on a log scale
param_grid = {
    'svc__kernel': ['rbf', 'linear'],
    'svc__C':      [0.01, 0.1, 1, 10, 100],
    'svc__gamma':  ['scale', 'auto', 0.001, 0.01, 0.1],
}

grid = GridSearchCV(
    pipe, param_grid,
    cv=5,
    scoring='accuracy',
    n_jobs=-1,
    verbose=0
)
grid.fit(X_train, y_train)

print("Best parameters: ", grid.best_params_)
print(f"Best CV accuracy: {grid.best_score_:.4f}")
print(f"Test accuracy:    {grid.score(X_test, y_test):.4f}")

# Show top 5 combinations
import pandas as pd
results = pd.DataFrame(grid.cv_results_)
top5 = results.nsmallest(5, 'rank_test_score')[
    ['param_svc__kernel', 'param_svc__C',
     'param_svc__gamma', 'mean_test_score']
]
print(top5.to_string(index=False))

LinearSVC — Faster for Large Datasets and Text

from sklearn.svm import LinearSVC
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.pipeline import Pipeline
from sklearn.datasets import fetch_20newsgroups
from sklearn.metrics import classification_report

# LinearSVC is much faster than SVC(kernel='linear') for large datasets
# Uses liblinear instead of libsvm — scales to millions of samples

categories = ['sci.space', 'comp.graphics',
              'rec.sport.hockey', 'talk.politics.guns']

train = fetch_20newsgroups(subset='train', categories=categories)
test  = fetch_20newsgroups(subset='test',  categories=categories)

text_pipeline = Pipeline([
    ('tfidf', TfidfVectorizer(
        sublinear_tf=True,
        max_features=50000,
        ngram_range=(1, 2),
        stop_words='english'
    )),
    ('clf', LinearSVC(
        C=1.0,
        max_iter=2000,
        random_state=42
    ))
])

text_pipeline.fit(train.data, train.target)
y_pred = text_pipeline.predict(test.data)

print(f"Test Accuracy: {text_pipeline.score(test.data, test.target):.4f}")
print(classification_report(
    test.target, y_pred,
    target_names=train.target_names
))

SVM for Regression — SVR

SVM can also do regression. Instead of finding a boundary that separates classes, SVR (Support Vector Regression) finds a tube of width 2ε that contains as many training points as possible while staying as flat as possible. Points inside the tube contribute zero loss. Points outside the tube are penalised.

from sklearn.svm import SVR
from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.metrics import r2_score, mean_absolute_error

housing = fetch_california_housing()
X, y = housing.data, housing.target

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# SVR — always needs StandardScaler
svr_pipe = Pipeline([
    ('scaler', StandardScaler()),
    ('svr', SVR(
        kernel='rbf',
        C=100,           # penalty for points outside tube
        epsilon=0.1,     # tube half-width — zero loss inside
        gamma='scale'
    ))
])

svr_pipe.fit(X_train, y_train)
y_pred = svr_pipe.predict(X_test)

print(f"R²  Score: {r2_score(y_test, y_pred):.4f}")
print(f"MAE:       ${mean_absolute_error(y_test, y_pred)*100_000:.0f}")
print(f"Support Vectors: {svr_pipe['svr'].n_support_[0]}")

Feature Importance with Linear SVM

Linear SVMs have a weight vector w that directly indicates how much each feature contributes to the decision boundary. This is equivalent to feature importance — the magnitude of each coefficient tells you how strongly that feature pushes a prediction toward one class.

from sklearn.svm import LinearSVC
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.datasets import load_breast_cancer
import numpy as np
import pandas as pd

cancer = load_breast_cancer()
X, y = cancer.data, cancer.target

pipe = Pipeline([
    ('scaler', StandardScaler()),
    ('svm',    LinearSVC(C=1.0, max_iter=5000, random_state=42))
])
pipe.fit(X, y)

# Coefficients — positive = pushes toward class 1 (malignant)
coef = pipe['svm'].coef_[0]

feat_df = pd.DataFrame({
    'feature':    cancer.feature_names,
    'coefficient': coef
}).sort_values('coefficient', key=lambda x: x.abs(), ascending=False)

print("Top features pushing toward MALIGNANT (positive coef):")
print(feat_df[feat_df['coefficient'] > 0].head(5).to_string(index=False))

print("\nTop features pushing toward BENIGN (negative coef):")
print(feat_df[feat_df['coefficient'] < 0].head(5).to_string(index=False))

SVM vs Other Classifiers — Complete Comparison

When to Use SVM

Strengths vs Weaknesses

Real-World Applications

Golden Rules