Accuracy, Precision, Recall, ROC-AUC & Choosing a Metric

def confusion_counts(predictions, labels):
    tp = sum(1 for p, y in zip(predictions, labels) if p == 1 and y == 1)
    fp = sum(1 for p, y in zip(predictions, labels) if p == 1 and y == 0)
    fn = sum(1 for p, y in zip(predictions, labels) if p == 0 and y == 1)
    tn = sum(1 for p, y in zip(predictions, labels) if p == 0 and y == 0)
    return tp, fp, fn, tn

def precision_recall_f1(predictions, labels):
    tp, fp, fn, tn = confusion_counts(predictions, labels)
    precision = tp / (tp + fp) if (tp + fp) else 0.0
    recall = tp / (tp + fn) if (tp + fn) else 0.0
    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0
    return precision, recall, f1

# 1 = fraud, 0 = not fraud. Model catches most fraud but has some false alarms.
predictions = [1, 0, 1, 1, 0, 0, 1, 0, 0, 0]
labels      = [1, 0, 0, 1, 0, 0, 1, 0, 1, 0]

precision, recall, f1 = precision_recall_f1(predictions, labels)
print(f"Precision: {precision:.2f}")
print(f"Recall:    {recall:.2f}")
print(f"F1:        {f1:.2f}")

# The "always predict not-fraud" trap from earlier in this chapter:
lazy_predictions = [0] * len(labels)
p2, r2, f2 = precision_recall_f1(lazy_predictions, labels)
print(f"\nAlways predicting 0 -> precision={p2:.2f}, recall={r2:.2f}, F1={f2:.2f}")
print("Recall (and F1) collapse to 0. Accuracy alone would have hidden this completely.")

def confusion_at_threshold(scores, labels, threshold):
    tp = sum(1 for s, y in zip(scores, labels) if s >= threshold and y == 1)
    fp = sum(1 for s, y in zip(scores, labels) if s >= threshold and y == 0)
    fn = sum(1 for s, y in zip(scores, labels) if s < threshold and y == 1)
    tn = sum(1 for s, y in zip(scores, labels) if s < threshold and y == 0)
    return tp, fp, fn, tn

def roc_points(scores, labels, n_thresholds=11):
    points = []
    for i in range(n_thresholds):
        threshold = i / (n_thresholds - 1)
        tp, fp, fn, tn = confusion_at_threshold(scores, labels, threshold)
        tpr = tp / (tp + fn) if (tp + fn) else 0.0
        fpr = fp / (fp + tn) if (fp + tn) else 0.0
        points.append((fpr, tpr))
    return sorted(points)

def approximate_auc(points):
    # Trapezoidal rule: sum the area of each trapezoid between consecutive points
    auc = 0.0
    for (x0, y0), (x1, y1) in zip(points, points[1:]):
        auc += (x1 - x0) * (y0 + y1) / 2
    return auc

# Model-predicted probabilities and true labels for 10 examples
scores = [0.9, 0.8, 0.7, 0.6, 0.55, 0.45, 0.4, 0.3, 0.2, 0.1]
labels = [1,   1,   0,   1,   1,    0,    0,   1,   0,   0]

points = roc_points(scores, labels)
auc = approximate_auc(points)

print("FPR, TPR at each threshold:")
for fpr, tpr in points:
    print(f"  FPR={fpr:.2f}, TPR={tpr:.2f}")
print(f"\nApproximate AUC: {auc:.3f}")
print("(0.5 is random guessing, 1.0 is perfect. This model is meaningfully better than random.)")

import random

def class_weighted_bce(predictions, labels, weight_for_positive):
    import math
    eps = 1e-12
    total = 0.0
    for p, y in zip(predictions, labels):
        p = min(max(p, eps), 1 - eps)
        weight = weight_for_positive if y == 1 else 1.0
        total += weight * (y * math.log(p) + (1 - y) * math.log(1 - p))
    return -total / len(predictions)

# 90 majority-class examples worth of signal, 10 minority-class examples
predictions = [0.4] * 9 + [0.6]  # mostly wrong on the one minority example
labels      = [0] * 9 + [1]

unweighted = class_weighted_bce(predictions, labels, weight_for_positive=1.0)
weighted = class_weighted_bce(predictions, labels, weight_for_positive=9.0)
print(f"Unweighted loss: {unweighted:.4f}")
print(f"Weighted loss (9x on the minority class): {weighted:.4f}")
print("Weighting makes the one minority mistake count as much as several majority ones.")

def smote_style_synthesize(minority_points, seed=42):
    rng = random.Random(seed)
    a, b = rng.sample(minority_points, 2)
    t = rng.random()  # interpolate somewhere between the two real points
    return [a[i] + t * (b[i] - a[i]) for i in range(len(a))]

minority_examples = [[45, 90], [50, 95], [48, 88]]  # from the KNN chapter's "buys premium" class
synthetic_point = smote_style_synthesize(minority_examples)
print(f"\nSynthetic minority point: {[round(v, 1) for v in synthetic_point]}")
print("A new example that plausibly belongs to the minority class, not a copy of any real one.")