Der Satz von Bayes in verständlicher Sprache

What Is Bayes' Theorem?

# Bayes' theorem components:
# P(H|X)  = Posterior  -- What we want to know
#                         'Probability of H given we observed X'
# P(X|H)  = Likelihood -- How likely is X if H is true?
# P(H)    = Prior      -- Our belief in H before seeing X
# P(X)    = Evidence   -- Overall probability of observing X

# Rearranged:
# Posterior = (Likelihood * Prior) / Evidence

print('Posterior = (Likelihood * Prior) / Evidence')

A Medical Test Example

# Medical test: Bayes' theorem applied
P_disease    = 0.01   # Prior: 1% have the disease
P_no_disease = 0.99   # 99% are healthy

P_pos_given_disease    = 0.95  # True positive rate (sensitivity)
P_pos_given_no_disease = 0.05  # False positive rate

# P(positive) = total probability of a positive test
P_positive = (P_pos_given_disease * P_disease +
              P_pos_given_no_disease * P_no_disease)

# Bayes: P(disease | positive)
P_disease_given_pos = (P_pos_given_disease * P_disease) / P_positive

print(f'P(disease | positive test) = {P_disease_given_pos:.2%}')
# Only ~16%! Not 95% -- the low prior dominates

Prior, Likelihood, and Posterior Explained

# Sequential Bayesian updating
# Start with 1% prior, observe 3 positive tests

prior = 0.01
P_pos_given_disease = 0.95
P_pos_given_no_disease = 0.05

for test_num in range(1, 4):
    likelihood_pos = P_pos_given_disease
    P_positive = likelihood_pos * prior + P_pos_given_no_disease * (1 - prior)
    posterior = (likelihood_pos * prior) / P_positive
    
    print(f'After test {test_num}: P(disease) = {posterior:.3%}')
    prior = posterior  # Posterior becomes new prior

# Three positive tests raise probability substantially

Bayes' Theorem for Classification

# Classification with Bayes' theorem
# Predict P(spam | email_features) vs P(ham | email_features)

# Prior (from training data)
P_spam = 0.3  # 30% of emails are spam
P_ham  = 0.7  # 70% are ham

# Likelihood: P(features | class) from training
# (simplified: single word 'offer' seen)
P_offer_given_spam = 0.6  # 'offer' appears in 60% of spam
P_offer_given_ham  = 0.1  # 'offer' appears in 10% of ham

# Unnormalised posteriors (ignore P(offer) -- same denominator)
posterior_spam = P_offer_given_spam * P_spam  # 0.18
posterior_ham  = P_offer_given_ham  * P_ham   # 0.07

print('Unnormalised: spam=', posterior_spam, 'ham=', posterior_ham)
print('Predicted class: spam' if posterior_spam > posterior_ham else 'ham')

The Law of Total Probability

# Law of total probability: P(X) = sum_c P(X|c) * P(c)
classes  = ['spam', 'ham', 'newsletter']
priors   = [0.3,    0.5,   0.2]          # Must sum to 1.0
P_word_given_class = [0.5, 0.1, 0.3]     # Likelihood of 'free' in each class

# Total probability of seeing the word 'free'
P_free = sum(lik * pri for lik, pri in zip(P_word_given_class, priors))
print(f'P(free) = {P_free}')

# Posteriors (normalised)
for cls, pri, lik in zip(classes, priors, P_word_given_class):
    posterior = lik * pri / P_free
    print(f'P({cls} | free) = {posterior:.3f}')

From Formula to Algorithm

# Naive Bayes algorithm in pseudocode

# Training:
# For each class c:
#   prior[c] = count(class==c) / total_samples
#   For each feature f:
#     likelihood[c][f] = P(feature_f | class==c)  # from training data

# Prediction for new sample x:
# For each class c:
#   log_prob[c] = log(prior[c])
#   For each feature f:
#     log_prob[c] += log(likelihood[c][f=x_f])  # naive: assume independence
# return argmax(log_prob)  # class with highest log-posterior

print('Naive independence assumption: P(f1,f2,...|c) = P(f1|c)*P(f2|c)*...*P(fn|c)')

Why Log Probabilities Avoid Underflow

import numpy as np

# Direct multiplication: underflows
probs = [0.1] * 100  # 100 features, each P=0.1
product = np.prod(probs)
print('Direct product:', product)  # 0.0 -- underflow!

# Log space: numerically stable
log_sum = np.sum(np.log(probs))
print('Log sum:', log_sum)         # -100 * log(10) -- valid

# Compare two classes
log_prob_A = np.sum(np.log([0.1] * 100))
log_prob_B = np.sum(np.log([0.2] * 100))
print('Class A log-prob:', log_prob_A)
print('Class B log-prob:', log_prob_B)
print('Predicted:', 'A' if log_prob_A > log_prob_B else 'B')

Bayes Optimal Classifier: The Gold Standard

# The Bayes error rate is the irreducible error
# Even a perfect model cannot beat it on a given distribution

# Example: predicting coin flip from noisy signal
import numpy as np

np.random.seed(42)
# True label: 50/50 coin
y_true = np.random.choice([0, 1], size=1000)
# Signal: 70% correlated with true label
signal = np.where(np.random.rand(1000) < 0.7, y_true, 1-y_true)

# Optimal prediction: just use the signal
acc = (signal == y_true).mean()
print(f'Optimal classifier accuracy: {acc:.3f}')
print(f'Bayes error rate (irreducible): {1-0.70:.3f} = 30%')

Real-World Applications of Bayes' Theorem

# Real-world Bayes: A/B test - which version is better?
# After 100 conversions from 1000 visitors for version A
# and 120 conversions from 1000 visitors for version B:

from scipy import stats

# Beta distribution as posterior for conversion rate
alpha_A, beta_A = 100 + 1, 900 + 1  # Beta(successes+1, failures+1)
alpha_B, beta_B = 120 + 1, 880 + 1

# Monte Carlo: P(B is better than A)
samples_A = stats.beta(alpha_A, beta_A).rvs(100000)
samples_B = stats.beta(alpha_B, beta_B).rvs(100000)

P_B_better = (samples_B > samples_A).mean()
print(f'P(B is better than A): {P_B_better:.3f}')

The Independence Assumption: When Is It Valid?

# When does naive independence work?
# Even with correlated features, naive bayes often ranks classes correctly

import numpy as np

# Two correlated features that both indicate spam
words = {'buy': 0, 'now': 1, 'click': 2, 'here': 3}

# Spam email often has all four; ham rarely does
# Independence assumption ignores that 'buy' and 'now' co-occur
# But the combined signal is still strong for detecting spam

P_word_spam = [0.7, 0.6, 0.5, 0.6]
P_word_ham  = [0.1, 0.1, 0.05, 0.1]

# For email with all 4 words
log_spam = np.sum(np.log(P_word_spam))
log_ham  = np.sum(np.log(P_word_ham))
print('Log P(features|spam):', log_spam.round(2))
print('Log P(features|ham): ', log_ham.round(2))
print('Predicted: SPAM' if log_spam > log_ham else 'HAM')

Bayes Factor: Comparing Hypotheses

# Bayes Factor example: comparing two models
# H0: coin is fair (p=0.5)
# H1: coin is biased (p=0.7)
# Observed: 8 heads in 10 flips

from scipy.stats import binom

observed_heads = 8
n_flips = 10

# Likelihood under each hypothesis
L_H0 = binom.pmf(observed_heads, n_flips, p=0.5)
L_H1 = binom.pmf(observed_heads, n_flips, p=0.7)

BF = L_H1 / L_H0
print(f'P(8 heads | fair coin) = {L_H0:.4f}')
print(f'P(8 heads | p=0.7)    = {L_H1:.4f}')
print(f'Bayes Factor (H1/H0)  = {BF:.2f}')
print('BF > 3: moderate evidence for biased coin')

Quick Check

Lesson Recap