LLM Evaluation & Benchmarks: GLUE, SuperGLUE, and MMLU

The Story That Makes Benchmarks Make Sense

LLM evaluation is the systematic process of measuring how well a language model performs across a defined set of tasks. Because language is infinitely varied, evaluation is hard — and the field has converged on a handful of trusted benchmark suites that have become the common currency of AI research.

What Are We Actually Measuring?

Before diving into specific benchmarks, understand the landscape of NLP capabilities that evaluation suites are designed to probe. Language understanding involves multiple distinct skills — and different benchmarks target different slices of that landscape.

GLUE — General Language Understanding Evaluation

GLUE (Wang et al., 2018) aggregates nine NLP classification tasks into a single overall score — a weighted average — and provides a public leaderboard for fair model comparison. All nine tasks test a model's ability to read a sentence (or pair of sentences) and make a classification decision.

The Nine GLUE Tasks

GLUE Score Evolution (2018–2020)

Running GLUE Evaluation in Python

from datasets import load_dataset
from transformers import (
    AutoTokenizer, AutoModelForSequenceClassification,
    TrainingArguments, Trainer
)
import numpy as np
import evaluate

# ── 1. Load a GLUE task (SST-2 sentiment) ──────────────────
dataset   = load_dataset("glue", "sst2")
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

# ── 2. Tokenise ─────────────────────────────────────────────
def tokenize(batch):
    return tokenizer(batch["sentence"], truncation=True,
                     padding="max_length", max_length=128)

tokenised = dataset.map(tokenize, batched=True)

# ── 3. Load model with classification head ──────────────────
model = AutoModelForSequenceClassification.from_pretrained(
    "bert-base-uncased", num_labels=2
)

# ── 4. Define GLUE metric ────────────────────────────────────
glue_metric = evaluate.load("glue", "sst2")

def compute_metrics(eval_pred):
    logits, labels = eval_pred
    preds = np.argmax(logits, axis=-1)
    return glue_metric.compute(predictions=preds, references=labels)

# ── 5. Train & evaluate ──────────────────────────────────────
args = TrainingArguments(
    output_dir="./sst2-bert",
    num_train_epochs=3,
    per_device_train_batch_size=32,
    evaluation_strategy="epoch",
    load_best_model_at_end=True,
    metric_for_best_model="accuracy",
    report_to="none"
)
trainer = Trainer(
    model=model, args=args,
    train_dataset=tokenised["train"],
    eval_dataset=tokenised["validation"],
    compute_metrics=compute_metrics
)
trainer.train()
results = trainer.evaluate()
print(f"SST-2 Accuracy: {results['eval_accuracy']:.4f}")

SuperGLUE — The Harder Successor

The Eight SuperGLUE Tasks

Evaluating on SuperGLUE — BoolQ

from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments
import evaluate, numpy as np

# ── SuperGLUE: BoolQ task (question + passage → yes/no) ─────
dataset   = load_dataset("super_glue", "boolq")
tokenizer = AutoTokenizer.from_pretrained("roberta-base")

def preprocess(batch):
    return tokenizer(
        batch["question"], batch["passage"],
        truncation=True, padding="max_length", max_length=256
    )

tokenised = dataset.map(preprocess, batched=True)
tokenised = tokenised.rename_column("label", "labels")
tokenised.set_format("torch", columns=["input_ids", "attention_mask", "labels"])

model  = AutoModelForSequenceClassification.from_pretrained("roberta-base", num_labels=2)
metric = evaluate.load("super_glue", "boolq")

def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions    = np.argmax(logits, axis=-1)
    return metric.compute(predictions=predictions, references=labels)

# SuperGLUE tasks are small — need more epochs at lower LR
args = TrainingArguments(
    output_dir="./boolq-roberta",
    num_train_epochs=10,
    per_device_train_batch_size=16,
    learning_rate=1e-5,
    evaluation_strategy="epoch",
    report_to="none"
)
trainer = Trainer(model=model, args=args,
                   train_dataset=tokenised["train"],
                   eval_dataset=tokenised["validation"],
                   compute_metrics=compute_metrics)
trainer.train()
print(trainer.evaluate())

MMLU — Massive Multitask Language Understanding

What an MMLU Question Looks Like

MMLU Zero-Shot Evaluation in Python

from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForCausalLM
import torch, numpy as np
from tqdm import tqdm

# ── MMLU is evaluated zero/few-shot — no fine-tuning ────────
model_name = "mistralai/Mistral-7B-Instruct-v0.1"
tokenizer  = AutoTokenizer.from_pretrained(model_name)
model      = AutoModelForCausalLM.from_pretrained(
    model_name, torch_dtype=torch.float16, device_map="auto"
)

subject = "clinical_knowledge"
dataset = load_dataset("cais/mmlu", subject, split="test")
CHOICES = ["A", "B", "C", "D"]

def build_prompt(row):
    # 5-shot prompt format from original MMLU paper
    q = row["question"]
    prompt = f"Question: {q}\n"
    for letter, choice in zip(CHOICES, row["choices"]):
        prompt += f"{letter}. {choice}\n"
    prompt += "Answer:"
    return prompt

def get_choice_logprob(prompt, choice):
    """Score each answer by its log-probability (higher = more likely)."""
    full_text = prompt + f" {choice}"
    inputs    = tokenizer(full_text, return_tensors="pt").to(model.device)
    with torch.no_grad():
        outputs = model(**inputs, labels=inputs["input_ids"])
    return -outputs.loss.item()

correct = 0
for row in tqdm(dataset, desc=f"MMLU {subject}"):
    prompt   = build_prompt(row)
    scores   = [get_choice_logprob(prompt, c) for c in CHOICES]
    pred_idx = int(np.argmax(scores))
    if pred_idx == row["answer"]:
        correct += 1

accuracy = correct / len(dataset)
print(f"\n{subject} accuracy : {accuracy:.4f}")
print(f"Random baseline  : 0.2500")
print(f"Expert human     : ~0.8900")

GLUE vs SuperGLUE vs MMLU — Direct Comparison

Full Multi-Benchmark Evaluation Pipeline

from datasets import load_dataset
from transformers import pipeline
import numpy as np

MODEL     = "HuggingFaceH4/zephyr-7b-beta"
generator = pipeline(
    "text-generation", model=MODEL,
    device_map="auto", torch_dtype="auto",
    max_new_tokens=4, temperature=0.0, do_sample=False
)

# ── GLUE SST-2 ───────────────────────────────────────────────
def eval_sst2(n=200):
    ds = load_dataset("glue", "sst2", split=f"validation[:{n}]")
    correct = 0
    for row in ds:
        prompt = (f"Classify as Positive or Negative.\nReview: {row['sentence']}\nSentiment:")
        out  = generator(prompt)[0]["generated_text"][len(prompt):].strip().lower()
        pred = 1 if "positive" in out else 0
        correct += int(pred == row["label"])
    return correct / n

# ── SuperGLUE BoolQ ──────────────────────────────────────────
def eval_boolq(n=200):
    ds = load_dataset("super_glue", "boolq", split=f"validation[:{n}]")
    correct = 0
    for row in ds:
        prompt = (f"Passage: {row['passage'][:400]}\nQuestion: {row['question']}\nAnswer yes or no:")
        out  = generator(prompt)[0]["generated_text"][len(prompt):].strip().lower()
        pred = 1 if "yes" in out else 0
        correct += int(pred == row["label"])
    return correct / n

# ── MMLU subset ──────────────────────────────────────────────
CHOICES = ["A", "B", "C", "D"]
def eval_mmlu(subject="high_school_biology", n=100):
    ds = load_dataset("cais/mmlu", subject, split=f"test[:{n}]")
    correct = 0
    for row in ds:
        opts   = "\n".join(f"{c}. {t}" for c,t in zip(CHOICES, row["choices"]))
        prompt = f"Question: {row['question']}\n{opts}\nAnswer:"
        out    = generator(prompt)[0]["generated_text"][len(prompt):].strip()[:1].upper()
        pred   = CHOICES.index(out) if out in CHOICES else -1
        correct += int(pred == row["answer"])
    return correct / n

# ── Report ────────────────────────────────────────────────────
results = {
    "GLUE SST-2":          eval_sst2(200),
    "SuperGLUE BoolQ":     eval_boolq(200),
    "MMLU HS Biology":     eval_mmlu("high_school_biology", 100),
    "MMLU Clinical Know.": eval_mmlu("clinical_knowledge",   100),
}
print("\n═══ Evaluation Report ═══")
for task, acc in results.items():
    bar = "█" * int(acc * 20) + "░" * (20 - int(acc * 20))
    print(f"{task:25s}  {bar}  {acc:.2%}")

Critical Limitations — What Benchmarks Cannot Tell You

Golden Rules — LLM Evaluation Practitioner Checklist